Study 1: 270 runs complete + judge validation pipeline

- config_study1.yaml: 3 models × 3 oversight × bare framing × n=30
- Judge validation: 54-run subset, gold=Sonnet 4.6, proxy=Grok 4.1 Fast (BB κ=0.702) + GPT-4.1 (GB κ=0.878)
- Fix OpenAI batch provider: BytesIO, method/url fields, response.body parsing
- Scripts: extract_subset.py, judge_validation.py
- Dissertation chapters updated (intro, methodology, results, conclusion)

3 files changed, 586 insertions, 5 deletions

diff --git a/scripts/extract_subset.py b/scripts/extract_subset.py
new file mode 100644
index 0000000..b32f5b3
--- /dev/null
+++ b/scripts/extract_subset.py
@@ -0,0 +1,163 @@
+"""
+Extract a stratified subset of runs from an experiment directory.
+
+Usage:
+    # Extract 20% subset from Study 1 (54 runs from 270)
+    python scripts/extract_subset.py \
+        --logs-dir logs/v2_study1 \
+        --output-dir logs/v2_study1_subset \
+        --subset-fraction 0.2
+
+    # Extract 10% subset
+    python scripts/extract_subset.py \
+        --logs-dir logs/v2_study1 \
+        --output-dir logs/v2_study1_subset \
+        --subset-fraction 0.1
+"""
+
+import argparse
+import os
+import random
+import shutil
+import sys
+from collections import defaultdict
+
+
+def discover_runs(logs_dir: str) -> dict[str, list[str]]:
+    """Discover all runs, grouped by (model, scenario, goal_type, oversight)."""
+    runs = defaultdict(list)
+
+    for root, dirs, files in os.walk(logs_dir):
+        basename = os.path.basename(root)
+
+        if basename == "baseline":
+            continue
+
+        for f in files:
+            if f.endswith(".json") and not f.startswith("_") and not f.startswith("."):
+                rel_path = os.path.relpath(root, logs_dir)
+                parts = rel_path.split(os.sep)
+
+                if len(parts) == 4:
+                    model, scenario, goal_type, oversight = parts
+                    key = (model, scenario, goal_type, oversight)
+                    runs[key].append(os.path.join(root, f))
+
+    return runs
+
+
+def extract_subset(
+    logs_dir: str,
+    output_dir: str,
+    subset_fraction: float,
+    seed: int = 42,
+) -> None:
+    """Extract a stratified random subset of runs."""
+    random.seed(seed)
+
+    runs_by_cell = discover_runs(logs_dir)
+
+    print(f"Found {len(runs_by_cell)} cells:")
+    total_runs = 0
+    for cell, paths in sorted(runs_by_cell.items()):
+        print(f"  {'/'.join(cell)}: {len(paths)} runs")
+        total_runs += len(paths)
+    print(f"Total: {total_runs} runs\n")
+
+    subset_runs = []
+    for cell, paths in runs_by_cell.items():
+        n_subset = max(1, int(len(paths) * subset_fraction))
+        selected = random.sample(paths, min(n_subset, len(paths)))
+        subset_runs.extend(selected)
+        print(f"  {'/'.join(cell)}: selected {len(selected)}/{len(paths)} runs")
+
+    print(f"\nTotal subset: {len(subset_runs)} runs\n")
+
+    copied_runs = 0
+    scenario_dirs = set()
+
+    for src_path in subset_runs:
+        rel_path = os.path.relpath(src_path, logs_dir)
+        dst_path = os.path.join(output_dir, rel_path)
+
+        os.makedirs(os.path.dirname(dst_path), exist_ok=True)
+        shutil.copy2(src_path, dst_path)
+        copied_runs += 1
+
+        cell_parts = rel_path.split(os.sep)
+        if len(cell_parts) >= 4:
+            src_scenario_dir = os.path.join(logs_dir, cell_parts[0], cell_parts[1])
+            dst_scenario_dir = os.path.join(output_dir, cell_parts[0], cell_parts[1])
+            scenario_dirs.add((src_scenario_dir, dst_scenario_dir))
+
+        if len(cell_parts) >= 4:
+            cell_str = "/".join(cell_parts[:4])
+        else:
+            cell_str = rel_path
+        print(f"  Copied: {cell_str}/{os.path.basename(src_path)}")
+
+    print(f"\nCopying baselines...")
+    for src_dir, dst_dir in sorted(scenario_dirs):
+        baseline_src = os.path.join(src_dir, "baseline.md")
+        baseline_dst = os.path.join(dst_dir, "baseline.md")
+        if os.path.exists(baseline_src):
+            os.makedirs(dst_dir, exist_ok=True)
+            shutil.copy2(baseline_src, baseline_dst)
+            print(f"  Copied baseline: {os.path.relpath(dst_dir, output_dir)}")
+        else:
+            print(f"  Warning: no baseline.md found at {src_dir}")
+
+    print(
+        f"\nDone. Copied {copied_runs} runs + {len(scenario_dirs)} baselines to {output_dir}"
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract stratified subset from experiment logs"
+    )
+    parser.add_argument(
+        "--logs-dir",
+        required=True,
+        help="Source directory containing experiment runs",
+    )
+    parser.add_argument(
+        "--output-dir",
+        required=True,
+        help="Output directory for subset",
+    )
+    parser.add_argument(
+        "--subset-fraction",
+        type=float,
+        default=0.2,
+        help="Fraction of runs to extract per cell (default: 0.2)",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed for reproducibility (default: 42)",
+    )
+
+    args = parser.parse_args()
+
+    if not os.path.isdir(args.logs_dir):
+        print(f"Error: {args.logs_dir} is not a directory")
+        sys.exit(1)
+
+    if args.subset_fraction <= 0 or args.subset_fraction > 1:
+        print(
+            f"Error: subset-fraction must be between 0 and 1, got {args.subset_fraction}"
+        )
+        sys.exit(1)
+
+    extract_subset(
+        logs_dir=args.logs_dir,
+        output_dir=args.output_dir,
+        subset_fraction=args.subset_fraction,
+        seed=args.seed,
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/judge_comparison.py b/scripts/judge_comparison.py
index b344480..506754f 100644
--- a/scripts/judge_comparison.py
+++ b/scripts/judge_comparison.py
@@ -194,13 +194,13 @@ def compare_two_csvs(csv1_path: str, csv2_path: str) -> Dict:
 def run_all_models(
     logs_dir: str, scenarios_dir: str, logs: List[str], output_dir: str
 ) -> tuple[List[str], List[tuple]]:
-    """Run all judge model pairs and save to separate CSVs."""
+    """Run all judge model pairs and save to separate CSVs. Skips models with existing CSVs."""
     models = [
-        ("claude-sonnet-4-20250514", "anthropic"),
-        ("gpt-4.1", "openai"),
+        ("claude-sonnet-4-6", "anthropic"),
         ("gpt-4.1-mini", "openai"),
-        ("claude-haiku-4-20250514", "anthropic"),
-        ("grok-fast-4.1", "xai"),
+        ("claude-haiku-4-5-20251001", "anthropic"),
+        ("grok-4-1-fast-reasoning", "xai"),
+        ("grok-4-1-fast-non-reasoning", "xai"),
     ]
 
     csv_paths = []
@@ -209,6 +209,10 @@ def run_all_models(
         output_path = os.path.join(output_dir, f"results_{safe_name}.csv")
         csv_paths.append(output_path)
 
+        if os.path.exists(output_path):
+            print(f"\n=== Skipping: {model_id} (CSV already exists: {output_path}) ===")
+            continue
+
         print(f"\n=== Running: {model_id} (both prongs) ===")
         verdicts = run_judgment(logs_dir, scenarios_dir, logs, model_id, provider)
         write_csv(output_path, verdicts)
diff --git a/scripts/judge_validation.py b/scripts/judge_validation.py
new file mode 100644
index 0000000..b3490bb
--- /dev/null
+++ b/scripts/judge_validation.py
@@ -0,0 +1,414 @@
+"""
+Judge Validation Tool - Find cheap proxy judges using inter-rate reliability.
+
+Workflow:
+    1. Run gold standard (expensive model) on subset in batch mode
+    2. Run candidate cheap models on same subset in batch mode
+    3. Compare each model against gold standard using Cohen's Kappa
+    4. Report which models meet the Kappa threshold (k > 0.6)
+
+Usage:
+    python scripts/judge_validation.py --logs-dir logs/v2_study1_subset --output results.csv
+
+Configuration:
+    Edit GOLD_STANDARD and CANDIDATE_MODELS at the top of this file.
+"""
+
+import argparse
+import csv
+import os
+import sys
+import time
+from collections import defaultdict
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.judge import Judge
+from src.judge.batch_providers import AnthropicBatchProvider, OpenAIBatchProvider, XAIBatchProvider
+
+# =============================================================================
+# CONFIGURATION - Edit these
+# =============================================================================
+
+GOLD_STANDARD = {
+    "blackbox": {
+        "id": "claude-sonnet-4-6",
+        "provider": "anthropic",
+        "temperature": 0,
+    },
+    "glassbox": {
+        "id": "claude-sonnet-4-6",
+        "provider": "anthropic",
+        "temperature": 0,
+    },
+}
+
+CANDIDATE_MODELS = [
+    {
+        "name": "gpt-4.1",
+        "blackbox": {"id": "gpt-4.1", "provider": "openai", "temperature": 0},
+        "glassbox": {"id": "gpt-4.1", "provider": "openai", "temperature": 0},
+    },
+    {
+        "name": "gpt-4.1-mini",
+        "blackbox": {"id": "gpt-4.1-mini", "provider": "openai", "temperature": 0},
+        "glassbox": {"id": "gpt-4.1-mini", "provider": "openai", "temperature": 0},
+    },
+    # {
+    #     "name": "claude-haiku-4-5",
+    #     "blackbox": {
+    #         "id": "claude-haiku-4-5",
+    #         "provider": "anthropic",
+    #         "temperature": 0,
+    #     },
+    #     "glassbox": {
+    #         "id": "claude-haiku-4-5",
+    #         "provider": "anthropic",
+    #         "temperature": 0,
+    #     },
+    # },
+    {
+        "name": "grok-4-1-fast-reasoning",
+        "blackbox": {
+            "id": "grok-4-1-fast-reasoning",
+            "provider": "xai",
+            "temperature": 0,
+        },
+        "glassbox": {
+            "id": "grok-4-1-fast-reasoning",
+            "provider": "xai",
+            "temperature": 0,
+        },
+    },
+    {
+        "name": "grok-4.20-reasoning",
+        "blackbox": {
+            "id": "grok-4.20-reasoning",
+            "provider": "xai",
+            "temperature": 0,
+        },
+        "glassbox": {
+            "id": "grok-4.20-reasoning",
+            "provider": "xai",
+            "temperature": 0,
+        },
+    },
+    {
+        "name": "grok-4.20-non-reasoning",
+        "blackbox": {
+            "id": "grok-4.20-non-reasoning",
+            "provider": "xai",
+            "temperature": 0,
+        },
+        "glassbox": {
+            "id": "grok-4.20-non-reasoning",
+            "provider": "xai",
+            "temperature": 0,
+        },
+    },
+]
+
+KAPPA_THRESHOLD = 0.6
+
+# =============================================================================
+
+PROVIDER_BATCH_CLASSES = {
+    "anthropic": AnthropicBatchProvider,
+    "openai": OpenAIBatchProvider,
+    "xai": XAIBatchProvider,
+}
+
+
+def discover_logs(logs_dir: str) -> list[str]:
+    """Discover all run JSON files in a logs directory."""
+    logs = []
+    for root, dirs, files in os.walk(logs_dir):
+        if os.path.basename(root) == "baseline":
+            continue
+        for f in files:
+            if f.endswith(".json") and not f.startswith("_"):
+                logs.append(os.path.join(root, f))
+    return sorted(logs)
+
+
+def run_batch_judgment(
+    logs_dir: str,
+    scenarios_dir: str,
+    model_config: dict,
+    output_path: str,
+) -> None:
+    """Run batch judgment with the given model config and save to CSV."""
+    print(f"\n  Running: {model_config['name']}")
+
+    batch_providers = {}
+    for prong in ["blackbox", "glassbox"]:
+        provider = model_config[prong]["provider"]
+        if provider in PROVIDER_BATCH_CLASSES and provider not in batch_providers:
+            batch_providers[provider] = PROVIDER_BATCH_CLASSES[provider]()
+
+    judge = Judge(
+        blackbox_model=model_config["blackbox"],
+        glassbox_model=model_config["glassbox"],
+        batch_providers=batch_providers,
+    )
+
+    log_files = discover_logs(logs_dir)
+    print(f"  Found {len(log_files)} logs")
+
+    batch_requests_by_provider, metadata_map = judge.prepare_batch_requests(
+        log_paths=log_files,
+        logs_dir=logs_dir,
+        scenarios_dir=scenarios_dir,
+        enabled_judges={"regex", "blackbox", "glassbox"},
+    )
+
+    total_requests = sum(len(reqs) for reqs in batch_requests_by_provider.values())
+    print(f"  {total_requests} API requests to submit")
+
+    batch_ids = judge.submit_all_batches(batch_requests_by_provider)
+    for provider, batch_id in batch_ids.items():
+        print(f"  {provider} batch: {batch_id}")
+
+    print(f"  Waiting for completion...")
+    judge.poll_all_batches(batch_ids)
+
+    verdicts = judge.collect_batch_results(batch_ids, metadata_map)
+
+    fieldnames = [
+        "run_id",
+        "model",
+        "scenario",
+        "oversight",
+        "regex",
+        "blackbox_category",
+        "blackbox_justification",
+        "glassbox_category",
+        "glassbox_sophistication",
+        "glassbox_justification",
+    ]
+
+    with open(output_path, "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for v in verdicts:
+            row = {
+                "run_id": v.get("run_id", ""),
+                "model": v.get("model", ""),
+                "scenario": v.get("scenario", ""),
+                "oversight": v.get("oversight", ""),
+                "regex": v.get("regex", ""),
+                "blackbox_category": v.get("blackbox", {}).get("category", ""),
+                "blackbox_justification": v.get("blackbox", {}).get(
+                    "justification", ""
+                ),
+                "glassbox_category": v.get("glassbox", {}).get("category", ""),
+                "glassbox_sophistication": v.get("glassbox", {}).get(
+                    "sophistication", ""
+                ),
+                "glassbox_justification": v.get("glassbox", {}).get(
+                    "justification", ""
+                ),
+            }
+            writer.writerow(row)
+
+    print(f"  Saved {len(verdicts)} verdicts to {output_path}")
+
+
+def compute_kappa(cats1: list[str], cats2: list[str]) -> float | None:
+    """Compute Cohen's Kappa between two raters' category assignments."""
+    from sklearn.metrics import cohen_kappa_score
+
+    valid = [(c1, c2) for c1, c2 in zip(cats1, cats2) if c1 and c2]
+    if len(valid) < 3:
+        return None
+
+    c1, c2 = zip(*valid)
+    try:
+        return cohen_kappa_score(c1, c2)
+    except Exception:
+        return None
+
+
+def compare_to_goldstandard(
+    gold_csv: str,
+    candidate_csv: str,
+    candidate_name: str,
+) -> dict:
+    """Compare candidate CSV to gold standard CSV."""
+    gold_verdicts = {}
+    with open(gold_csv, newline="") as f:
+        for row in csv.DictReader(f):
+            gold_verdicts[row["run_id"]] = row
+
+    cand_verdicts = {}
+    with open(candidate_csv, newline="") as f:
+        for row in csv.DictReader(f):
+            cand_verdicts[row["run_id"]] = row
+
+    run_ids = sorted(set(gold_verdicts.keys()) & set(cand_verdicts.keys()))
+
+    bb_cats_gold = [gold_verdicts[rid]["blackbox_category"] for rid in run_ids]
+    bb_cats_cand = [cand_verdicts[rid]["blackbox_category"] for rid in run_ids]
+
+    gb_cats_gold = [gold_verdicts[rid]["glassbox_category"] for rid in run_ids]
+    gb_cats_cand = [cand_verdicts[rid]["glassbox_category"] for rid in run_ids]
+
+    soph_gold = [gold_verdicts[rid]["glassbox_sophistication"] for rid in run_ids]
+    soph_cand = [cand_verdicts[rid]["glassbox_sophistication"] for rid in run_ids]
+
+    return {
+        "name": candidate_name,
+        "n": len(run_ids),
+        "blackbox_kappa": compute_kappa(bb_cats_gold, bb_cats_cand),
+        "glassbox_kappa": compute_kappa(gb_cats_gold, gb_cats_cand),
+        "sophistication_kappa": compute_kappa(soph_gold, soph_cand),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Judge Validation Tool")
+    parser.add_argument(
+        "--logs-dir",
+        required=True,
+        help="Directory containing experiment logs (subset)",
+    )
+    parser.add_argument(
+        "--scenarios-dir",
+        default="scenarios",
+        help="Directory containing scenario definitions",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="judge_validation",
+        help="Output directory for results CSVs",
+    )
+    args = parser.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    gold_name = f"gold_{GOLD_STANDARD['blackbox']['id']}"
+    gold_csv = os.path.join(args.output_dir, f"{gold_name}.csv")
+
+    if not os.path.exists(gold_csv):
+        print(f"\n{'=' * 60}")
+        print("STEP 1: Running gold standard")
+        print(f"{'=' * 60}")
+        run_batch_judgment(
+            args.logs_dir,
+            args.scenarios_dir,
+            {"name": gold_name, **GOLD_STANDARD},
+            gold_csv,
+        )
+    else:
+        print(f"\nGold standard already exists: {gold_csv}")
+
+    print(f"\n{'=' * 60}")
+    print("STEP 2: Running candidate models")
+    print(f"{'=' * 60}")
+
+    results = []
+    for model in CANDIDATE_MODELS:
+        name = model["name"]
+        csv_path = os.path.join(args.output_dir, f"{name}.csv")
+
+        if os.path.exists(csv_path):
+            print(f"\n  Skipping {name} (already exists)")
+            candidate_result = compare_to_goldstandard(gold_csv, csv_path, name)
+        else:
+            print(f"\n{'=' * 60}")
+            print(f"  Running: {name}")
+            print(f"{'=' * 60}")
+            run_batch_judgment(
+                args.logs_dir,
+                args.scenarios_dir,
+                model,
+                csv_path,
+            )
+            candidate_result = compare_to_goldstandard(gold_csv, csv_path, name)
+
+        results.append(candidate_result)
+
+        bb_k = (
+            f"{candidate_result['blackbox_kappa']:.3f}"
+            if candidate_result["blackbox_kappa"]
+            else "N/A"
+        )
+        gb_k = (
+            f"{candidate_result['glassbox_kappa']:.3f}"
+            if candidate_result["glassbox_kappa"]
+            else "N/A"
+        )
+        sp_k = (
+            f"{candidate_result['sophistication_kappa']:.3f}"
+            if candidate_result["sophistication_kappa"]
+            else "N/A"
+        )
+
+        bb_pass = (
+            candidate_result["blackbox_kappa"]
+            and candidate_result["blackbox_kappa"] > KAPPA_THRESHOLD
+        )
+        gb_pass = (
+            candidate_result["glassbox_kappa"]
+            and candidate_result["glassbox_kappa"] > KAPPA_THRESHOLD
+        )
+
+        print(f"\n  {name}:")
+        print(f"    Blackbox Kappa:     {bb_k} {'✓' if bb_pass else '✗'}")
+        print(f"    Glassbox Kappa:     {gb_k} {'✓' if gb_pass else '✗'}")
+        print(f"    Sophistication Kappa:{sp_k}")
+
+    print(f"\n{'=' * 60}")
+    print("SUMMARY")
+    print(f"{'=' * 60}")
+    print(f"{'Model':<30} {'BB κ':>8} {'GB κ':>8} {'Sph κ':>8} {'Pass':>6}")
+    print("-" * 60)
+
+    for r in results:
+        bb = r["blackbox_kappa"]
+        gb = r["glassbox_kappa"]
+        bb_str = f"{bb:.3f}" if bb else "N/A"
+        gb_str = f"{gb:.3f}" if gb else "N/A"
+        sp_str = (
+            f"{r['sophistication_kappa']:.3f}" if r["sophistication_kappa"] else "N/A"
+        )
+        pass_str = (
+            "YES"
+            if (bb and bb > KAPPA_THRESHOLD and gb and gb > KAPPA_THRESHOLD)
+            else "no"
+        )
+        print(f"{r['name']:<30} {bb_str:>8} {gb_str:>8} {sp_str:>8} {pass_str:>6}")
+
+    summary_csv = os.path.join(args.output_dir, "summary.csv")
+    with open(summary_csv, "w", newline="") as f:
+        writer = csv.DictWriter(
+            f,
+            fieldnames=[
+                "name",
+                "n",
+                "blackbox_kappa",
+                "glassbox_kappa",
+                "sophistication_kappa",
+                "pass",
+            ],
+        )
+        writer.writeheader()
+        for r in results:
+            writer.writerow(
+                {
+                    **r,
+                    "pass": "YES"
+                    if (
+                        r["blackbox_kappa"]
+                        and r["glassbox_kappa"]
+                        and r["blackbox_kappa"] > KAPPA_THRESHOLD
+                        and r["glassbox_kappa"] > KAPPA_THRESHOLD
+                    )
+                    else "no",
+                }
+            )
+    print(f"\nSummary saved to {summary_csv}")
+
+
+if __name__ == "__main__":
+    main()