summaryrefslogtreecommitdiff
path: root/scripts/judge_validation.py
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/judge_validation.py')
-rw-r--r--scripts/judge_validation.py414
1 files changed, 414 insertions, 0 deletions
diff --git a/scripts/judge_validation.py b/scripts/judge_validation.py
new file mode 100644
index 0000000..b3490bb
--- /dev/null
+++ b/scripts/judge_validation.py
@@ -0,0 +1,414 @@
+"""
+Judge Validation Tool - Find cheap proxy judges using inter-rate reliability.
+
+Workflow:
+ 1. Run gold standard (expensive model) on subset in batch mode
+ 2. Run candidate cheap models on same subset in batch mode
+ 3. Compare each model against gold standard using Cohen's Kappa
+ 4. Report which models meet the Kappa threshold (k > 0.6)
+
+Usage:
+ python scripts/judge_validation.py --logs-dir logs/v2_study1_subset --output results.csv
+
+Configuration:
+ Edit GOLD_STANDARD and CANDIDATE_MODELS at the top of this file.
+"""
+
+import argparse
+import csv
+import os
+import sys
+import time
+from collections import defaultdict
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.judge import Judge
+from src.judge.batch_providers import AnthropicBatchProvider, OpenAIBatchProvider, XAIBatchProvider
+
+# =============================================================================
+# CONFIGURATION - Edit these
+# =============================================================================
+
+GOLD_STANDARD = {
+ "blackbox": {
+ "id": "claude-sonnet-4-6",
+ "provider": "anthropic",
+ "temperature": 0,
+ },
+ "glassbox": {
+ "id": "claude-sonnet-4-6",
+ "provider": "anthropic",
+ "temperature": 0,
+ },
+}
+
+CANDIDATE_MODELS = [
+ {
+ "name": "gpt-4.1",
+ "blackbox": {"id": "gpt-4.1", "provider": "openai", "temperature": 0},
+ "glassbox": {"id": "gpt-4.1", "provider": "openai", "temperature": 0},
+ },
+ {
+ "name": "gpt-4.1-mini",
+ "blackbox": {"id": "gpt-4.1-mini", "provider": "openai", "temperature": 0},
+ "glassbox": {"id": "gpt-4.1-mini", "provider": "openai", "temperature": 0},
+ },
+ # {
+ # "name": "claude-haiku-4-5",
+ # "blackbox": {
+ # "id": "claude-haiku-4-5",
+ # "provider": "anthropic",
+ # "temperature": 0,
+ # },
+ # "glassbox": {
+ # "id": "claude-haiku-4-5",
+ # "provider": "anthropic",
+ # "temperature": 0,
+ # },
+ # },
+ {
+ "name": "grok-4-1-fast-reasoning",
+ "blackbox": {
+ "id": "grok-4-1-fast-reasoning",
+ "provider": "xai",
+ "temperature": 0,
+ },
+ "glassbox": {
+ "id": "grok-4-1-fast-reasoning",
+ "provider": "xai",
+ "temperature": 0,
+ },
+ },
+ {
+ "name": "grok-4.20-reasoning",
+ "blackbox": {
+ "id": "grok-4.20-reasoning",
+ "provider": "xai",
+ "temperature": 0,
+ },
+ "glassbox": {
+ "id": "grok-4.20-reasoning",
+ "provider": "xai",
+ "temperature": 0,
+ },
+ },
+ {
+ "name": "grok-4.20-non-reasoning",
+ "blackbox": {
+ "id": "grok-4.20-non-reasoning",
+ "provider": "xai",
+ "temperature": 0,
+ },
+ "glassbox": {
+ "id": "grok-4.20-non-reasoning",
+ "provider": "xai",
+ "temperature": 0,
+ },
+ },
+]
+
+KAPPA_THRESHOLD = 0.6
+
+# =============================================================================
+
+PROVIDER_BATCH_CLASSES = {
+ "anthropic": AnthropicBatchProvider,
+ "openai": OpenAIBatchProvider,
+ "xai": XAIBatchProvider,
+}
+
+
+def discover_logs(logs_dir: str) -> list[str]:
+ """Discover all run JSON files in a logs directory."""
+ logs = []
+ for root, dirs, files in os.walk(logs_dir):
+ if os.path.basename(root) == "baseline":
+ continue
+ for f in files:
+ if f.endswith(".json") and not f.startswith("_"):
+ logs.append(os.path.join(root, f))
+ return sorted(logs)
+
+
+def run_batch_judgment(
+ logs_dir: str,
+ scenarios_dir: str,
+ model_config: dict,
+ output_path: str,
+) -> None:
+ """Run batch judgment with the given model config and save to CSV."""
+ print(f"\n Running: {model_config['name']}")
+
+ batch_providers = {}
+ for prong in ["blackbox", "glassbox"]:
+ provider = model_config[prong]["provider"]
+ if provider in PROVIDER_BATCH_CLASSES and provider not in batch_providers:
+ batch_providers[provider] = PROVIDER_BATCH_CLASSES[provider]()
+
+ judge = Judge(
+ blackbox_model=model_config["blackbox"],
+ glassbox_model=model_config["glassbox"],
+ batch_providers=batch_providers,
+ )
+
+ log_files = discover_logs(logs_dir)
+ print(f" Found {len(log_files)} logs")
+
+ batch_requests_by_provider, metadata_map = judge.prepare_batch_requests(
+ log_paths=log_files,
+ logs_dir=logs_dir,
+ scenarios_dir=scenarios_dir,
+ enabled_judges={"regex", "blackbox", "glassbox"},
+ )
+
+ total_requests = sum(len(reqs) for reqs in batch_requests_by_provider.values())
+ print(f" {total_requests} API requests to submit")
+
+ batch_ids = judge.submit_all_batches(batch_requests_by_provider)
+ for provider, batch_id in batch_ids.items():
+ print(f" {provider} batch: {batch_id}")
+
+ print(f" Waiting for completion...")
+ judge.poll_all_batches(batch_ids)
+
+ verdicts = judge.collect_batch_results(batch_ids, metadata_map)
+
+ fieldnames = [
+ "run_id",
+ "model",
+ "scenario",
+ "oversight",
+ "regex",
+ "blackbox_category",
+ "blackbox_justification",
+ "glassbox_category",
+ "glassbox_sophistication",
+ "glassbox_justification",
+ ]
+
+ with open(output_path, "w", newline="") as f:
+ writer = csv.DictWriter(f, fieldnames=fieldnames)
+ writer.writeheader()
+ for v in verdicts:
+ row = {
+ "run_id": v.get("run_id", ""),
+ "model": v.get("model", ""),
+ "scenario": v.get("scenario", ""),
+ "oversight": v.get("oversight", ""),
+ "regex": v.get("regex", ""),
+ "blackbox_category": v.get("blackbox", {}).get("category", ""),
+ "blackbox_justification": v.get("blackbox", {}).get(
+ "justification", ""
+ ),
+ "glassbox_category": v.get("glassbox", {}).get("category", ""),
+ "glassbox_sophistication": v.get("glassbox", {}).get(
+ "sophistication", ""
+ ),
+ "glassbox_justification": v.get("glassbox", {}).get(
+ "justification", ""
+ ),
+ }
+ writer.writerow(row)
+
+ print(f" Saved {len(verdicts)} verdicts to {output_path}")
+
+
+def compute_kappa(cats1: list[str], cats2: list[str]) -> float | None:
+ """Compute Cohen's Kappa between two raters' category assignments."""
+ from sklearn.metrics import cohen_kappa_score
+
+ valid = [(c1, c2) for c1, c2 in zip(cats1, cats2) if c1 and c2]
+ if len(valid) < 3:
+ return None
+
+ c1, c2 = zip(*valid)
+ try:
+ return cohen_kappa_score(c1, c2)
+ except Exception:
+ return None
+
+
+def compare_to_goldstandard(
+ gold_csv: str,
+ candidate_csv: str,
+ candidate_name: str,
+) -> dict:
+ """Compare candidate CSV to gold standard CSV."""
+ gold_verdicts = {}
+ with open(gold_csv, newline="") as f:
+ for row in csv.DictReader(f):
+ gold_verdicts[row["run_id"]] = row
+
+ cand_verdicts = {}
+ with open(candidate_csv, newline="") as f:
+ for row in csv.DictReader(f):
+ cand_verdicts[row["run_id"]] = row
+
+ run_ids = sorted(set(gold_verdicts.keys()) & set(cand_verdicts.keys()))
+
+ bb_cats_gold = [gold_verdicts[rid]["blackbox_category"] for rid in run_ids]
+ bb_cats_cand = [cand_verdicts[rid]["blackbox_category"] for rid in run_ids]
+
+ gb_cats_gold = [gold_verdicts[rid]["glassbox_category"] for rid in run_ids]
+ gb_cats_cand = [cand_verdicts[rid]["glassbox_category"] for rid in run_ids]
+
+ soph_gold = [gold_verdicts[rid]["glassbox_sophistication"] for rid in run_ids]
+ soph_cand = [cand_verdicts[rid]["glassbox_sophistication"] for rid in run_ids]
+
+ return {
+ "name": candidate_name,
+ "n": len(run_ids),
+ "blackbox_kappa": compute_kappa(bb_cats_gold, bb_cats_cand),
+ "glassbox_kappa": compute_kappa(gb_cats_gold, gb_cats_cand),
+ "sophistication_kappa": compute_kappa(soph_gold, soph_cand),
+ }
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Judge Validation Tool")
+ parser.add_argument(
+ "--logs-dir",
+ required=True,
+ help="Directory containing experiment logs (subset)",
+ )
+ parser.add_argument(
+ "--scenarios-dir",
+ default="scenarios",
+ help="Directory containing scenario definitions",
+ )
+ parser.add_argument(
+ "--output-dir",
+ default="judge_validation",
+ help="Output directory for results CSVs",
+ )
+ args = parser.parse_args()
+
+ os.makedirs(args.output_dir, exist_ok=True)
+
+ gold_name = f"gold_{GOLD_STANDARD['blackbox']['id']}"
+ gold_csv = os.path.join(args.output_dir, f"{gold_name}.csv")
+
+ if not os.path.exists(gold_csv):
+ print(f"\n{'=' * 60}")
+ print("STEP 1: Running gold standard")
+ print(f"{'=' * 60}")
+ run_batch_judgment(
+ args.logs_dir,
+ args.scenarios_dir,
+ {"name": gold_name, **GOLD_STANDARD},
+ gold_csv,
+ )
+ else:
+ print(f"\nGold standard already exists: {gold_csv}")
+
+ print(f"\n{'=' * 60}")
+ print("STEP 2: Running candidate models")
+ print(f"{'=' * 60}")
+
+ results = []
+ for model in CANDIDATE_MODELS:
+ name = model["name"]
+ csv_path = os.path.join(args.output_dir, f"{name}.csv")
+
+ if os.path.exists(csv_path):
+ print(f"\n Skipping {name} (already exists)")
+ candidate_result = compare_to_goldstandard(gold_csv, csv_path, name)
+ else:
+ print(f"\n{'=' * 60}")
+ print(f" Running: {name}")
+ print(f"{'=' * 60}")
+ run_batch_judgment(
+ args.logs_dir,
+ args.scenarios_dir,
+ model,
+ csv_path,
+ )
+ candidate_result = compare_to_goldstandard(gold_csv, csv_path, name)
+
+ results.append(candidate_result)
+
+ bb_k = (
+ f"{candidate_result['blackbox_kappa']:.3f}"
+ if candidate_result["blackbox_kappa"]
+ else "N/A"
+ )
+ gb_k = (
+ f"{candidate_result['glassbox_kappa']:.3f}"
+ if candidate_result["glassbox_kappa"]
+ else "N/A"
+ )
+ sp_k = (
+ f"{candidate_result['sophistication_kappa']:.3f}"
+ if candidate_result["sophistication_kappa"]
+ else "N/A"
+ )
+
+ bb_pass = (
+ candidate_result["blackbox_kappa"]
+ and candidate_result["blackbox_kappa"] > KAPPA_THRESHOLD
+ )
+ gb_pass = (
+ candidate_result["glassbox_kappa"]
+ and candidate_result["glassbox_kappa"] > KAPPA_THRESHOLD
+ )
+
+ print(f"\n {name}:")
+ print(f" Blackbox Kappa: {bb_k} {'✓' if bb_pass else '✗'}")
+ print(f" Glassbox Kappa: {gb_k} {'✓' if gb_pass else '✗'}")
+ print(f" Sophistication Kappa:{sp_k}")
+
+ print(f"\n{'=' * 60}")
+ print("SUMMARY")
+ print(f"{'=' * 60}")
+ print(f"{'Model':<30} {'BB κ':>8} {'GB κ':>8} {'Sph κ':>8} {'Pass':>6}")
+ print("-" * 60)
+
+ for r in results:
+ bb = r["blackbox_kappa"]
+ gb = r["glassbox_kappa"]
+ bb_str = f"{bb:.3f}" if bb else "N/A"
+ gb_str = f"{gb:.3f}" if gb else "N/A"
+ sp_str = (
+ f"{r['sophistication_kappa']:.3f}" if r["sophistication_kappa"] else "N/A"
+ )
+ pass_str = (
+ "YES"
+ if (bb and bb > KAPPA_THRESHOLD and gb and gb > KAPPA_THRESHOLD)
+ else "no"
+ )
+ print(f"{r['name']:<30} {bb_str:>8} {gb_str:>8} {sp_str:>8} {pass_str:>6}")
+
+ summary_csv = os.path.join(args.output_dir, "summary.csv")
+ with open(summary_csv, "w", newline="") as f:
+ writer = csv.DictWriter(
+ f,
+ fieldnames=[
+ "name",
+ "n",
+ "blackbox_kappa",
+ "glassbox_kappa",
+ "sophistication_kappa",
+ "pass",
+ ],
+ )
+ writer.writeheader()
+ for r in results:
+ writer.writerow(
+ {
+ **r,
+ "pass": "YES"
+ if (
+ r["blackbox_kappa"]
+ and r["glassbox_kappa"]
+ and r["blackbox_kappa"] > KAPPA_THRESHOLD
+ and r["glassbox_kappa"] > KAPPA_THRESHOLD
+ )
+ else "no",
+ }
+ )
+ print(f"\nSummary saved to {summary_csv}")
+
+
+if __name__ == "__main__":
+ main()
generated by cgit v1.2.3 (git 2.46.0) at 2026-09-26 11:07:10 +0000