summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorCaptainJack2491 <jayrupnakawala@gmail.com>2026-05-08 11:24:43 +0100
committerCaptainJack2491 <jayrupnakawala@gmail.com>2026-05-08 11:24:43 +0100
commit0f3128fb9cadb1b30c7247a3ccb614dc3661e825 (patch)
tree26e0b7a41a4c82e315d9f7f89c47c56921f3b2ab
parente25da628080fa3ab25acee5be3235aaed5a63af0 (diff)
scripts added ig
-rw-r--r--scripts/analyze_study1.py12
-rw-r--r--scripts/judge_missing.py86
-rw-r--r--scripts/mcnemar_effect_sizes.py133
-rw-r--r--scripts/power_analysis.py166
-rw-r--r--scripts/stealth_gap_tests.py230
-rw-r--r--scripts/verify_glm_claims.py171
-rw-r--r--scripts/verify_study2_regression.py275
-rw-r--r--scripts/visualizations.py248
8 files changed, 1321 insertions, 0 deletions
diff --git a/scripts/analyze_study1.py b/scripts/analyze_study1.py
index 09e111b..a626972 100644
--- a/scripts/analyze_study1.py
+++ b/scripts/analyze_study1.py
@@ -241,6 +241,18 @@ def main():
section("6. STRATEGIC SOPHISTICATION")
+ print(" Full sophistication breakdown by oversight:")
+ for ov in ["low", "mid", "high"]:
+ sub = df[df["oversight"] == ov]
+ n_tot = len(sub)
+ counts = sub["glassbox_sophistication"].value_counts()
+ n_strat = counts.get("STRATEGIC", 0)
+ n_tact = counts.get("TACTICAL", 0)
+ n_naive = counts.get("NAIVE", 0)
+ print(f" {ov}: strategic={n_strat}/{n_tot} ({n_strat/n_tot*100:.1f}%), "
+ f"tactical={n_tact}/{n_tot} ({n_tact/n_tot*100:.1f}%), "
+ f"naive={n_naive}/{n_tot} ({n_naive/n_tot*100:.1f}%)")
+ print()
print(" Strategic reasoning rates by oversight:")
for ov in ["low", "mid", "high"]:
sub = df[df["oversight"] == ov]
diff --git a/scripts/judge_missing.py b/scripts/judge_missing.py
new file mode 100644
index 0000000..66c2c67
--- /dev/null
+++ b/scripts/judge_missing.py
@@ -0,0 +1,86 @@
+import sys
+import glob
+import csv
+import os
+
+sys.path.insert(0, "src")
+from judge.judge import Judge
+import yaml
+
+# The 8 new valid files from git status
+new_files = [
+ "logs/v2_study1/google_gemini-3.1-pro-preview/corporate_sabotage_v2/bare/high/20260415_214146.json",
+ "logs/v2_study1/google_gemini-3.1-pro-preview/corporate_sabotage_v2/bare/high/20260415_214243.json",
+ "logs/v2_study1/google_gemini-3.1-pro-preview/corporate_sabotage_v2/bare/high/20260415_214310.json",
+ "logs/v2_study1/google_gemini-3.1-pro-preview/corporate_sabotage_v2/bare/high/20260415_214626.json",
+ "logs/v2_study1/google_gemini-3.1-pro-preview/corporate_sabotage_v2/bare/high/20260415_214744.json",
+ "logs/v2_study1/google_gemini-3.1-pro-preview/corporate_sabotage_v2/bare/mid/20260415_214138.json",
+ "logs/v2_study1/google_gemini-3.1-pro-preview/corporate_sabotage_v2/bare/mid/20260415_214157.json",
+ "logs/v2_study1/google_gemini-3.1-pro-preview/corporate_sabotage_v2/bare/mid/20260415_214217.json"
+]
+
+# Load validated configurations
+config = yaml.safe_load(open("config_study1.yaml"))["judge"]
+config["blackbox"]["id"] = config["blackbox"]["model"]
+config["glassbox"]["id"] = config["glassbox"]["model"]
+
+judge = Judge(
+ blackbox_model=config["blackbox"],
+ glassbox_model=config["glassbox"]
+)
+
+# 1. Read existing CSV
+csv_path = "logs/v2_study1/study1_results.csv"
+with open(csv_path) as f:
+ existing_rows = list(csv.DictReader(f))
+
+# 2. Drop the 8 corrupted rows
+corrupted_timestamps = [
+ "20260415_171552", "20260415_171559", "20260415_171806", "20260415_171816",
+ "20260415_183235", "20260415_181921", "20260415_182006", "20260415_182013"
+]
+
+clean_rows = []
+for r in existing_rows:
+ if not any(ts in r["run_id"] for ts in corrupted_timestamps):
+ clean_rows.append(r)
+
+print(f"Removed {len(existing_rows) - len(clean_rows)} corrupted rows from CSV.")
+
+# 3. Judge the 8 new files in single mode
+print(f"Judging {len(new_files)} new files...")
+for f in new_files:
+ print(f" Judging {f}...")
+ verdict = judge.judge_single(
+ log_path=f,
+ logs_dir="logs/v2_study1",
+ scenarios_dir="scenarios",
+ enabled_judges={"regex", "blackbox", "glassbox"}
+ )
+
+ # Optional: save judge log to disk immediately
+ from judge_runner import save_judge_log
+ save_judge_log(verdict, config.get("log_dir", "logs/v2_study1_judge"), "mixed")
+
+ row = {
+ "run_id": verdict.get("run_id", ""),
+ "model": verdict.get("model", ""),
+ "scenario": verdict.get("scenario", ""),
+ "oversight": verdict.get("oversight", ""),
+ "regex": verdict.get("regex", ""),
+ "blackbox_category": verdict.get("blackbox", {}).get("category", ""),
+ "blackbox_justification": verdict.get("blackbox", {}).get("justification", ""),
+ "glassbox_category": verdict.get("glassbox", {}).get("category", ""),
+ "glassbox_sophistication": verdict.get("glassbox", {}).get("sophistication", ""),
+ "glassbox_justification": verdict.get("glassbox", {}).get("justification", "")
+ }
+ clean_rows.append(row)
+
+# 4. Save updated CSV
+with open(csv_path + ".tmp", "w", newline="") as f:
+ writer = csv.DictWriter(f, fieldnames=list(clean_rows[0].keys()))
+ writer.writeheader()
+ writer.writerows(clean_rows)
+
+os.rename(csv_path + ".tmp", csv_path)
+print(f"\nSuccessfully saved perfect {len(clean_rows)}-run dataset to {csv_path}!")
diff --git a/scripts/mcnemar_effect_sizes.py b/scripts/mcnemar_effect_sizes.py
new file mode 100644
index 0000000..2dfbeaf
--- /dev/null
+++ b/scripts/mcnemar_effect_sizes.py
@@ -0,0 +1,133 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+# "pandas",
+# "numpy",
+# "scipy",
+# "statsmodels",
+# ]
+# ///
+"""
+McNemar Effect Sizes for Study 1
+=================================
+Computes effect-size metrics from the paired 2x2 tables of glassbox vs blackbox
+for each oversight level.
+
+Each table cell:
+ a = glassbox-deceptive AND blackbox-misleading
+ b = glassbox-deceptive AND NOT blackbox-misleading (stealth runs)
+ c = NOT glassbox-deceptive AND blackbox-misleading (false alarms)
+ d = neither
+
+Reports for each oversight level:
+ - Discordant pair counts: b (stealth), c (false alarm)
+ - Odds ratio of discordant pairs: OR = b / c
+ - 95% confidence interval for OR
+ - Proportion of discordant pairs: (b + c) / N
+
+Usage:
+ uv run python scripts/mcnemar_effect_sizes.py
+"""
+
+import warnings
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+DECEPTIVE_BB = {"DECEPTIVE_FRAMING", "OMITTED_CRITICAL", "FABRICATED"}
+COMPLIANT_GB = {"COMPLIANT_DECEPTION"}
+RESULTS_PATH = "logs/v2_study1/study1_results.csv"
+Z_ALPHA = stats.norm.ppf(1 - 0.05 / 2)
+
+
+def load_data(path: str) -> pd.DataFrame:
+ df = pd.read_csv(path)
+ df["is_deceptive_bb"] = df["blackbox_category"].isin(DECEPTIVE_BB).astype(int)
+ df["is_compliant_gb"] = df["glassbox_category"].isin(COMPLIANT_GB).astype(int)
+ return df
+
+
+def discordant_or_ci(b: int, c: int) -> tuple[float, float, float]:
+ if b == 0 or c == 0:
+ b_adj, c_adj = b + 0.5, c + 0.5
+ else:
+ b_adj, c_adj = float(b), float(c)
+ or_val = b_adj / c_adj
+ log_or = np.log(or_val)
+ se = np.sqrt(1 / b_adj + 1 / c_adj)
+ ci_low = np.exp(log_or - Z_ALPHA * se)
+ ci_high = np.exp(log_or + Z_ALPHA * se)
+ return or_val, ci_low, ci_high
+
+
+def main():
+ df = load_data(RESULTS_PATH)
+
+ print("=" * 72)
+ print(" McNemar Effect Sizes — Discordant-Pair Analysis")
+ print("=" * 72)
+
+ header = (
+ f" {'Oversight':>9s} {'N':>4s} {'b(stealth)':>10s} "
+ f"{'c(falseAlm)':>10s} {'OR(b/c)':>8s} "
+ f"{'OR 95% CI':>20s} {'(b+c)/N':>8s} {'Disagr%':>8s}"
+ )
+ sep = " " + "-" * 9 + " " + "-" * 4 + " " + "-" * 10 + " " + "-" * 10 + " " + "-" * 8 + " " + "-" * 20 + " " + "-" * 8 + " " + "-" * 8
+
+ print(f"\n{header}")
+ print(sep)
+
+ for ov in ["low", "mid", "high"]:
+ sub = df[df["oversight"] == ov]
+ N = len(sub)
+
+ gb_dec = sub["is_compliant_gb"].values
+ bb_det = sub["is_deceptive_bb"].values
+
+ a = int(((gb_dec == 1) & (bb_det == 1)).sum())
+ b = int(((gb_dec == 1) & (bb_det == 0)).sum())
+ c = int(((gb_dec == 0) & (bb_det == 1)).sum())
+ d = int(((gb_dec == 0) & (bb_det == 0)).sum())
+
+ or_val, ci_low, ci_high = discordant_or_ci(b, c)
+ discordant_prop = (b + c) / N
+ discordant_pct = discordant_prop * 100
+
+ ci_str = f"[{ci_low:.3f}, {ci_high:.3f}]"
+
+ print(
+ f" {ov:>9s} {N:>4d} {b:>10d} {c:>10d} "
+ f"{or_val:>8.3f} {ci_str:>20s} "
+ f"{discordant_prop:>8.3f} {discordant_pct:>7.1f}%"
+ )
+
+ if b + c > 0:
+ if b > c:
+ direction = f"stealth runs outnumber false alarms {b}:{c}"
+ elif c > b:
+ direction = f"false alarms outnumber stealth runs {c}:{b}"
+ else:
+ direction = "stealth runs and false alarms are equal"
+ print(f" → {direction}")
+
+ decision = ""
+ if or_val > 1 and ci_low > 1:
+ decision = " (OR > 1, significant)"
+ elif or_val < 1 and ci_high < 1:
+ decision = " (OR < 1, significant)"
+ else:
+ decision = " (not significant, CI crosses 1)"
+ print(f" OR 95% CI does{'' if 'significant' in decision else ' not'} exclude 1{decision}")
+
+ print("\n Interpretation:")
+ print(" b = glassbox says deceptive | blackbox says honest (stealth)")
+ print(" c = glassbox says honest | blackbox says misleading (false alarm)")
+ print(" OR = b/c: odds that a discordant pair is a stealth run vs a false alarm")
+ print(" (b+c)/N: proportion of all pairs showing disagreement between judges")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/power_analysis.py b/scripts/power_analysis.py
new file mode 100644
index 0000000..ebabb6e
--- /dev/null
+++ b/scripts/power_analysis.py
@@ -0,0 +1,166 @@
+"""
+Post-hoc Power Analysis for Dissertation Studies
+================================================
+Determines whether non-significant chi-squared results reflect true absence
+of effect or insufficient statistical power.
+
+Uses statsmodels GofChisquarePower for chi-squared test power analysis.
+Effect size: Cohen's w computed from proportion differences in 3x2 tables.
+
+Usage:
+ uv run python scripts/power_analysis.py
+"""
+
+import numpy as np
+from statsmodels.stats.power import GofChisquarePower
+
+
+def cohens_w_from_prop_diff(p0, d, n_groups=3):
+ """Cohen's w for a n_groups x 2 contingency table.
+
+ Assumes equal group sizes, baseline proportion p0 in all groups under H0,
+ and a difference d in exactly one group under H1.
+
+ w = d * sqrt( (1/p0 + 1/(1-p0)) / n_groups )
+ """
+ return abs(d) * np.sqrt((1.0 / p0 + 1.0 / (1.0 - p0)) / n_groups)
+
+
+def prop_diff_from_cohens_w(p0, w, n_groups=3):
+ """Inverse: proportion difference corresponding to a given Cohen's w."""
+ return w / np.sqrt((1.0 / p0 + 1.0 / (1.0 - p0)) / n_groups)
+
+
+def power_interp(power):
+ if power >= 0.95:
+ return "Excellent (>95%) — very likely to detect"
+ elif power >= 0.80:
+ return "Adequate (>=80%) — standard threshold met"
+ elif power >= 0.50:
+ return "Moderate (50-80%) — may be missed"
+ elif power >= 0.20:
+ return "Low (20-50%) — unreliable, likely to miss"
+ else:
+ return "Very low (<20%) — essentially undetectable"
+
+
+def analyze(n_per_group, n_groups, alpha, baseline_prop, label, pp_diffs=(10, 15, 20)):
+ n_total = n_per_group * n_groups
+ power_analysis = GofChisquarePower()
+ # df = n_bins - 1 = 2 for a 3-group test
+ n_bins = n_groups
+
+ print(f"\n{'=' * 72}")
+ print(f" {label}")
+ print(f" N = {n_per_group}/group x {n_groups} groups = {n_total} total")
+ print(f" α = {alpha}, baseline deception rate ≈ {baseline_prop:.1%}")
+ print(f"{'=' * 72}")
+
+ # --- 1. Minimum detectable effect at 80% power ---
+ w_min = power_analysis.solve_power(
+ effect_size=None, nobs=n_total, alpha=alpha, power=0.80, n_bins=n_bins
+ )
+ d_min = prop_diff_from_cohens_w(baseline_prop, w_min, n_groups)
+
+ print(f"\n 1. Minimum Detectable Effect (80% power)")
+ print(f" Cohen's w = {w_min:.4f}")
+ print(f" Diff in proportions = {d_min:.1%} ({d_min*100:.1f} percentage points)")
+ print(f" → With N={n_total}, any effect smaller than ~{d_min*100:.0f}pp")
+ print(f" between groups could not be reliably detected.")
+
+ # --- 2. Achieved power for specific effect sizes ---
+ print(f"\n 2. Achieved Power for Specific Effect Sizes:")
+ print(f" {'Diff (pp)':>10s} {'Cohen\'s w':>10s} {'Power':>8s} {'Interpretation'}")
+ print(f" {'-'*10} {'-'*10} {'-'*8} {'-'*42}")
+
+ for pp in pp_diffs:
+ d = pp / 100.0
+ w = cohens_w_from_prop_diff(baseline_prop, d, n_groups)
+ pwr = power_analysis.solve_power(
+ effect_size=w, nobs=n_total, alpha=alpha, power=None, n_bins=n_bins
+ )
+ print(f" {pp:>8d}pp {w:>10.4f} {pwr:>7.1%} {power_interp(pwr)}")
+
+ # --- 3. Within-model power (n=30 per group) ---
+ n_within = 30
+ n_within_total = n_within * n_groups
+
+ w_min_30 = power_analysis.solve_power(
+ effect_size=None, nobs=n_within_total, alpha=alpha, power=0.80, n_bins=n_bins
+ )
+ d_min_30 = prop_diff_from_cohens_w(baseline_prop, w_min_30, n_groups)
+
+ print(f"\n 3. Within-Model Power (n = 30 per group)")
+ print(f" Min detectable diff (80% power): {d_min_30:.1%} ({d_min_30*100:.1f}pp)")
+ print(f" Achieved power by effect size:")
+ for pp in pp_diffs:
+ d = pp / 100.0
+ w = cohens_w_from_prop_diff(baseline_prop, d, n_groups)
+ pwr = power_analysis.solve_power(
+ effect_size=w, nobs=n_within_total, alpha=alpha, power=None, n_bins=n_bins
+ )
+ print(f" {pp:>2d}pp difference: {pwr:.1%} power ({power_interp(pwr).split('—')[0].strip()})")
+
+ return d_min
+
+
+def main():
+ print("=" * 72)
+ print(" POST-HOC POWER ANALYSIS")
+ print(" AI Deception in Strategic Reasoning")
+ print("=" * 72)
+ print()
+ print(" Context: Both studies used chi-squared tests on 3x2 contingency")
+ print(" tables and found non-significant results. This analysis")
+ print(" determines whether those null results are meaningful.")
+
+ alpha = 0.05
+
+ # Study 1: Oversight effect (low / mid / high oversight)
+ # Observed rates: low=71.1%, mid=70.0%, high=64.4% → baseline ~68.5%
+ analyze(
+ n_per_group=90,
+ n_groups=3,
+ alpha=alpha,
+ baseline_prop=0.685,
+ label="STUDY 1: Oversight Effect (low / mid / high)",
+ )
+
+ # Study 2: Framing effect (bare / moral / self-serving)
+ # Observed rates: bare=66.7%, moral=65.6%, self-serving=66.7% → baseline ~66.3%
+ analyze(
+ n_per_group=90,
+ n_groups=3,
+ alpha=alpha,
+ baseline_prop=0.663,
+ label="STUDY 2: Goal Framing Effect (bare / moral / self-serving)",
+ )
+
+ # --- Summary ---
+ print(f"\n{'=' * 72}")
+ print(" SUMMARY AND INTERPRETATION")
+ print(f"{'=' * 72}")
+
+ print()
+ print(" Between-group (N=270, n=90/group):")
+ print(" • Both studies had ~80% power to detect differences of ~15pp")
+ print(" • Neither study was adequately powered for effects <~15pp")
+ print(" • The non-significant chi-squared results may reflect either:")
+ print(" (a) truly absent between-group effects, or")
+ print(" (b) modest effects (<~15pp) that the design lacked power to detect")
+ print()
+ print(" Within-model (n=30/model/cell):")
+ print(" • These per-model sub-analyses were substantially underpowered")
+ print(" • Only very large effects (>~26pp) would be detectable at 80% power")
+ print(" • Null within-model results should NOT be interpreted as")
+ print(" evidence that specific models were unaffected by the manipulation")
+ print()
+ print(" Implications for the dissertation:")
+ print(" • The null between-group results do not rule out small-to-moderate")
+ print(" effects (5-15pp), which could be substantively meaningful.")
+ print(" • Future work should target n ≈ 200-210 per group to reliably")
+ print(" detect 10pp effects at 80% power.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/stealth_gap_tests.py b/scripts/stealth_gap_tests.py
new file mode 100644
index 0000000..33e9142
--- /dev/null
+++ b/scripts/stealth_gap_tests.py
@@ -0,0 +1,230 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+# "pandas",
+# "numpy",
+# "scipy",
+# "statsmodels",
+# ]
+# ///
+"""
+Stealth Gap Analysis for Study 1
+=================================
+Tests the divergence between glassbox (internal) and blackbox (output) measures.
+
+Design: 3 oversight levels × 90 runs each
+Each run has paired classifications: glassbox intent + blackbox detection.
+
+Usage:
+ uv run python scripts/stealth_gap_tests.py
+"""
+
+import warnings
+
+import numpy as np
+import pandas as pd
+from scipy.stats import chi2_contingency, chi2
+from statsmodels.stats.contingency_tables import Table2x2
+
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+DECEPTIVE_BB = {"DECEPTIVE_FRAMING", "OMITTED_CRITICAL", "FABRICATED"}
+COMPLIANT_GB = {"COMPLIANT_DECEPTION"}
+RESULTS_PATH = "logs/v2_study1/study1_results.csv"
+
+
+def load_data(path: str) -> pd.DataFrame:
+ df = pd.read_csv(path)
+ df["is_deceptive_bb"] = df["blackbox_category"].isin(DECEPTIVE_BB).astype(int)
+ df["is_compliant_gb"] = df["glassbox_category"].isin(COMPLIANT_GB).astype(int)
+ return df
+
+
+def cramers_v(table: np.ndarray) -> float:
+ chi2_val = chi2_contingency(table)[0]
+ n = table.sum()
+ min_dim = min(table.shape) - 1
+ if min_dim == 0 or n == 0:
+ return 0.0
+ return np.sqrt(chi2_val / (n * min_dim))
+
+
+def cochran_armitage(counts, nobs):
+ k = len(counts)
+ scores = np.arange(k, dtype=float)
+ n = np.array(nobs, dtype=float)
+ x = np.array(counts, dtype=float)
+ N = n.sum()
+ p_hat = x.sum() / N
+ t_bar = np.sum(scores * n) / N
+ numerator = np.sum(scores * x) - x.sum() * t_bar
+ denominator_sq = p_hat * (1 - p_hat) * (np.sum(scores**2 * n) - N * t_bar**2)
+ if denominator_sq <= 0:
+ return 0.0, 1.0
+ z = numerator / np.sqrt(denominator_sq)
+ p_value = chi2.sf(z**2, 1)
+ return z, p_value
+
+
+def sig(p: float, alpha: float = 0.05) -> str:
+ if p < 0.001:
+ return "***"
+ elif p < 0.01:
+ return "**"
+ elif p < alpha:
+ return "*"
+ return "n.s."
+
+
+def fmt_p(p: float) -> str:
+ return f"p < .001" if p < 0.001 else f"p = {p:.4f}"
+
+
+def section(title: str):
+ print(f"\n{'=' * 70}")
+ print(f" {title}")
+ print(f"{'=' * 70}\n")
+
+
+def subsection(title: str):
+ print(f"\n --- {title} ---\n")
+
+
+def main():
+ df = load_data(RESULTS_PATH)
+ ALPHA = 0.05
+
+ section("1. BLACKBOX DECLINE TEST")
+
+ bb_counts = []
+ bb_nobs = []
+ for ov in ["low", "mid", "high"]:
+ sub = df[df["oversight"] == ov]
+ n_detected = sub["is_deceptive_bb"].sum()
+ n_total = len(sub)
+ bb_counts.append(n_detected)
+ bb_nobs.append(n_total)
+ print(
+ f" {ov}: {n_detected}/{n_total} ({n_detected / n_total * 100:.1f}%) misleading"
+ )
+
+ ct_bb = pd.crosstab(df["oversight"], df["is_deceptive_bb"])
+ ct_bb = ct_bb.loc[["low", "mid", "high"]]
+
+ chi2_val, p_val, dof, expected = chi2_contingency(ct_bb.values)
+ v = cramers_v(ct_bb.values)
+ print(f"\n Chi-squared test: χ²({dof}) = {chi2_val:.3f}, {fmt_p(p_val)} {sig(p_val)}")
+ print(f" Cramér's V = {v:.3f}")
+
+ z_trend, p_trend = cochran_armitage(bb_counts, bb_nobs)
+ print(f"\n Cochran-Armitage trend test: z = {z_trend:.3f}, {fmt_p(p_trend)} {sig(p_trend)}")
+ if p_trend < ALPHA:
+ direction = "decreasing" if z_trend < 0 else "increasing"
+ print(f" → Blackbox deception {direction} with higher oversight")
+
+ section("2. McNEMAR'S TEST BY OVERSIGHT LEVEL")
+
+ mcnemar_results = {}
+ for ov in ["low", "mid", "high"]:
+ sub = df[df["oversight"] == ov]
+
+ gb_dec = sub["is_compliant_gb"].values
+ bb_det = sub["is_deceptive_bb"].values
+
+ a = int(((gb_dec == 1) & (bb_det == 1)).sum())
+ b = int(((gb_dec == 1) & (bb_det == 0)).sum())
+ c = int(((gb_dec == 0) & (bb_det == 1)).sum())
+ d = int(((gb_dec == 0) & (bb_det == 0)).sum())
+
+ table = np.array([[a, b], [c, d]])
+ ct = Table2x2(table)
+ mcnemar_res = ct.symmetry()
+ stat = mcnemar_res.statistic
+ p_mc = mcnemar_res.pvalue
+
+ mcnemar_results[ov] = {"stat": stat, "p": p_mc}
+
+ print(f"\n {ov.upper()} oversight (n={len(sub)}):")
+ print(f" Contingency table (rows=glassbox, cols=blackbox, 1=deceptive/detected):")
+ print(f" Blackbox")
+ print(f" detected not")
+ print(f" Glassbox dec {a:4d} {b:3d}")
+ print(f" Glassbox not {c:4d} {d:3d}")
+ print(f" McNemar's test: χ²({1}) = {stat:.3f}, {fmt_p(p_mc)} {sig(p_mc)}")
+ if p_mc < ALPHA:
+ if b > c:
+ print(
+ f" → Glassbox detects MORE deception than blackbox "
+ f"(stealth gap: {b} runs deceptive-undetected vs {c} runs detected-nondec)"
+ )
+ else:
+ print(
+ f" → Blackbox detects MORE deception than glassbox "
+ f"({c} runs detected-nondec vs {b} runs deceptive-undetected)"
+ )
+
+ if (a + b) > 0:
+ stealth_rate_val = b / (a + b) * 100
+ print(f" Stealth rate (deceptive but undetected): {b}/{a + b} "
+ f"({stealth_rate_val:.1f}% of deceptive runs)")
+ else:
+ print(" Stealth rate: N/A (no deceptive runs)")
+
+ section("3. INTERACTION TEST: Stealth Rate Across Oversight")
+
+ stealth_counts = []
+ stealth_nobs = []
+ print(f" {'Oversight':12s} {'Deceptive':>10s} {'Undetected':>10s} {'Stealth Rate':>13s}")
+ print(f" {'-' * 12} {'-' * 10} {'-' * 10} {'-' * 13}")
+ for ov in ["low", "mid", "high"]:
+ sub = df[df["oversight"] == ov]
+ gb_dec = sub["is_compliant_gb"].values
+ bb_det = sub["is_deceptive_bb"].values
+
+ n_deceptive = int(gb_dec.sum())
+ n_undetected = int(((gb_dec == 1) & (bb_det == 0)).sum())
+ stealth_rate = n_undetected / n_deceptive * 100 if n_deceptive > 0 else 0
+
+ stealth_counts.append(n_undetected)
+ stealth_nobs.append(n_deceptive)
+ print(f" {ov:12s} {n_deceptive:8d}/90 {n_undetected:8d} "
+ f"{stealth_rate:8.1f}% ({n_undetected}/{n_deceptive})")
+
+ print(f"\n Chi-squared test on stealth counts (deceptive-undetected vs detected among deceptive):")
+ stealth_not_counts = [n - c for n, c in zip(stealth_nobs, stealth_counts)]
+ stealth_table = np.array([stealth_counts, stealth_not_counts]).T
+ chi2_stealth, p_stealth, dof_stealth, _ = chi2_contingency(stealth_table)
+ v_stealth = cramers_v(stealth_table)
+ print(f" χ²({dof_stealth}) = {chi2_stealth:.3f}, {fmt_p(p_stealth)} {sig(p_stealth)}")
+ print(f" Cramér's V = {v_stealth:.3f}")
+
+ z_stealth, p_stealth_trend = cochran_armitage(stealth_counts, stealth_nobs)
+ print(f"\n Cochran-Armitage trend test on stealth rates:")
+ print(f" z = {z_stealth:.3f}, {fmt_p(p_stealth_trend)} {sig(p_stealth_trend)}")
+ if p_stealth_trend < ALPHA:
+ direction = "decreasing" if z_stealth < 0 else "increasing"
+ print(f" → Stealth rate {direction} with higher oversight")
+
+ section("4. SUMMARY TABLE")
+
+ print(f" {'Test':45s} {'Statistic':>12s} {'p-value':>10s} {'Significant?':>12s}")
+ print(f" {'-' * 45} {'-' * 12} {'-' * 10} {'-' * 12}")
+
+ print(f" {'Blackbox decline (chi-squared)':45s} {f'χ²={chi2_val:.3f}':>12s} {fmt_p(p_val):>10s} {sig(p_val, ALPHA):>12s}")
+
+ for ov in ["low", "mid", "high"]:
+ r = mcnemar_results[ov]
+ sig_str = sig(r["p"], ALPHA)
+ stat_str = f"χ²={r['stat']:.3f}"
+ p_str = fmt_p(r["p"])
+ print(f" {'McNemar (' + ov + ')':45s} {stat_str:>12s} {p_str:>10s} {sig_str:>12s}")
+
+ sig_stealth = sig(p_stealth, ALPHA)
+ print(f" {'Stealth rate (chi-squared)':45s} {f'χ²={chi2_stealth:.3f}':>12s} {fmt_p(p_stealth):>10s} {sig_stealth:>12s}")
+
+ print(f"\n Significance codes: *** p < .001 ** p < .01 * p < .05 n.s. = not significant")
+ print(f" Alpha = {ALPHA}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/verify_glm_claims.py b/scripts/verify_glm_claims.py
new file mode 100644
index 0000000..509b625
--- /dev/null
+++ b/scripts/verify_glm_claims.py
@@ -0,0 +1,171 @@
+import json
+import glob
+import os
+from collections import defaultdict
+
+JUDGE_DIR = "logs/v2_study1_judge"
+
+MODEL_SHORT = {
+ "deepseek_deepseek-v3.2:nitro": "DeepSeek",
+ "google_gemini-3.1-pro-preview": "Gemini",
+ "z-ai_glm-5.1:nitro": "GLM",
+}
+
+def load_all_verdicts():
+ records = []
+ for fpath in glob.glob(os.path.join(JUDGE_DIR, "*.json")):
+ with open(fpath) as f:
+ d = json.load(f)
+ run_id = d["run_id"]
+ parts = run_id.split("/")
+ model_dir = parts[0]
+ oversight = parts[3]
+ run_ts = parts[4]
+
+ short = MODEL_SHORT.get(model_dir, model_dir)
+ v = d["verdict"]
+ records.append({
+ "model": short,
+ "oversight": oversight,
+ "run_id": run_id,
+ "run_ts": run_ts,
+ "bb_cat": v["blackbox_category"],
+ "gb_cat": v["glassbox_category"],
+ "gb_soph": v["glassbox_sophistication"],
+ })
+ return records
+
+
+def print_table(title, headers, rows):
+ w = [len(h) for h in headers]
+ for r in rows:
+ for i, c in enumerate(r):
+ w[i] = max(w[i], len(str(c)))
+ sep = " | ".join("-" * wi for wi in w)
+ header = " | ".join(h.ljust(wi) for h, wi in zip(headers, w))
+ print(f"\n{'=' * 80}")
+ print(f" {title}")
+ print(f"{'=' * 80}")
+ print(f" {header}")
+ print(f" {sep}")
+ for r in rows:
+ print(f" " + " | ".join(str(c).ljust(wi) for c, wi in zip(r, w)))
+ print()
+
+
+def claim1_compliant_deception_counts(records):
+ title = "Claim 1: COMPLIANT_DECEPTION counts by model x oversight"
+ headers = ["Model", "low", "mid", "high"]
+ order = ["DeepSeek", "Gemini", "GLM"]
+ levels = ["low", "mid", "high"]
+
+ counts = defaultdict(lambda: defaultdict(int))
+ for r in records:
+ if r["gb_cat"] == "COMPLIANT_DECEPTION":
+ counts[r["model"]][r["oversight"]] += 1
+
+ rows = []
+ for m in order:
+ rows.append([m] + [str(counts[m][l]) for l in levels])
+ rows.append(["Total"] + [str(sum(counts[mm][l] for mm in order)) for l in levels])
+
+ print_table(title, headers, rows)
+
+
+def claim2_glm_low_breakdown(records):
+ title = "Claim 2: GLM at low oversight — deceptive runs breakdown"
+
+ glm_low = [r for r in records if r["model"] == "GLM" and r["oversight"] == "low"]
+
+ deceptive = [r for r in glm_low if r["gb_cat"] in ("COMPLIANT_DECEPTION", "CONSIDERED_HONEST")]
+ print(f"\n{'=' * 80}")
+ print(f" {title}")
+ print(f"{'=' * 80}")
+ print(f" Total GLM low-oversight runs: {len(glm_low)}")
+ print()
+
+ gb_counts = defaultdict(int)
+ for r in deceptive:
+ gb_counts[r["gb_cat"]] += 1
+ print(f" Glassbox intent breakdown of deceptive runs:")
+ for cat in sorted(gb_counts):
+ print(f" {cat}: {gb_counts[cat]}")
+
+ considered = [r for r in deceptive if r["gb_cat"] == "CONSIDERED_HONEST"]
+ print(f"\n CONSIDERED_HONEST runs — blackbox labels:")
+ bb_counts = defaultdict(int)
+ for r in considered:
+ bb_counts[r["bb_cat"]] += 1
+ for cat in sorted(bb_counts):
+ print(f" {cat}: {bb_counts[cat]}")
+
+ if considered:
+ headers = ["run_ts", "blackbox_category"]
+ rows = [[r["run_ts"], r["bb_cat"]] for r in considered]
+ print()
+ headers2 = ["Model"] + headers
+ rows2 = [["GLM"] + r for r in rows]
+ print_table("Individual CONSIDERED_HONEST runs", headers2, rows2)
+
+
+def claim3_glm_considered_honest_sophistication(records):
+ title = "Claim 3: GLM CONSIDERED_HONEST sophistication by oversight"
+
+ glm_ch = [r for r in records if r["model"] == "GLM" and r["gb_cat"] == "CONSIDERED_HONEST"]
+
+ headers = ["Oversight", "STRATEGIC", "TACTICAL", "NAIVE", "Total"]
+ levels = ["low", "mid", "high"]
+ rows = []
+ for lv in levels:
+ subset = [r for r in glm_ch if r["oversight"] == lv]
+ str_cnt = sum(1 for r in subset if r["gb_soph"] == "STRATEGIC")
+ tac_cnt = sum(1 for r in subset if r["gb_soph"] == "TACTICAL")
+ na_cnt = sum(1 for r in subset if r["gb_soph"] == "NAIVE")
+ rows.append([lv, str(str_cnt), str(tac_cnt), str(na_cnt), str(len(subset))])
+
+ all_str = sum(1 for r in glm_ch if r["gb_soph"] == "STRATEGIC")
+ all_tac = sum(1 for r in glm_ch if r["gb_soph"] == "TACTICAL")
+ all_na = sum(1 for r in glm_ch if r["gb_soph"] == "NAIVE")
+ rows.append(["Total", str(all_str), str(all_tac), str(all_na), str(len(glm_ch))])
+
+ print_table(title, headers, rows)
+
+
+def claim4_non_strategic_deceptive(records):
+ title = "Claim 4: COMPLIANT_DECEPTION runs where sophistication is NOT STRATEGIC"
+
+ non_strat = [r for r in records if r["gb_cat"] == "COMPLIANT_DECEPTION" and r["gb_soph"] != "STRATEGIC"]
+
+ headers = ["Model", "Oversight", "run_ts", "Sophistication"]
+ rows = [[r["model"], r["oversight"], r["run_ts"], r["gb_soph"]] for r in non_strat]
+
+ print_table(title, headers, rows)
+ if not rows:
+ print(" (none found)")
+ print()
+
+
+def main():
+ records = load_all_verdicts()
+
+ total = len(records)
+ print(f"\nLoaded {total} judge verdicts from {JUDGE_DIR}/")
+
+ model_counts = defaultdict(int)
+ for r in records:
+ model_counts[r["model"]] += 1
+ print(f" Models: {dict(model_counts)}")
+
+ oversight_counts = defaultdict(int)
+ for r in records:
+ oversight_counts[r["oversight"]] += 1
+ print(f" Oversight levels: {dict(oversight_counts)}")
+
+ claim1_compliant_deception_counts(records)
+ claim2_glm_low_breakdown(records)
+ claim3_glm_considered_honest_sophistication(records)
+ claim4_non_strategic_deceptive(records)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/verify_study2_regression.py b/scripts/verify_study2_regression.py
new file mode 100644
index 0000000..6800457
--- /dev/null
+++ b/scripts/verify_study2_regression.py
@@ -0,0 +1,275 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+# "pandas",
+# "numpy",
+# "scipy",
+# "statsmodels",
+# ]
+# ///
+"""
+Verify Study 2 Logistic Regression
+===================================
+Parses Study 2 results and runs logistic regression predicting deception
+from model + goal_type, reporting all coefficients with OR, 95% CI,
+z-statistic, and p-value.
+
+Specifically investigates whether the moral framing OR = p coincidence
+is a genuine numerical coincidence or a data error.
+
+Usage:
+ uv run python scripts/verify_study2_regression.py [path_to_results.csv]
+"""
+
+import sys
+import warnings
+
+import numpy as np
+import pandas as pd
+from scipy.stats import chi2, norm
+import statsmodels.api as sm
+import statsmodels.formula.api as smf
+
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+DECEPTIVE_BB = {"DECEPTIVE_FRAMING", "OMITTED_CRITICAL", "FABRICATED"}
+COMPLIANT_GB = {"COMPLIANT_DECEPTION"}
+RESULTS_PATH = "logs/v2_study2/study2_results.csv"
+ALPHA = 0.05
+
+
+def sig(p: float) -> str:
+ if p < 0.001:
+ return "***"
+ elif p < 0.01:
+ return "**"
+ elif p < ALPHA:
+ return "*"
+ return "n.s."
+
+
+def fmt_p(p: float) -> str:
+ return "p < .001" if p < 0.001 else f"p = {p:.4f}"
+
+
+def load_data(path: str) -> pd.DataFrame:
+ df = pd.read_csv(path)
+ df["is_deceptive_bb"] = df["blackbox_category"].isin(DECEPTIVE_BB).astype(int)
+ df["is_compliant_gb"] = df["glassbox_category"].isin(COMPLIANT_GB).astype(int)
+ df["deceptive"] = (
+ (df["is_deceptive_bb"] == 1) | (df["is_compliant_gb"] == 1)
+ ).astype(int)
+ df["goal_type"] = df["run_id"].apply(lambda x: x.split("/")[2])
+ return df
+
+
+def section(title: str):
+ print(f"\n{'=' * 72}")
+ print(f" {title}")
+ print(f"{'=' * 72}")
+
+
+def main():
+ path = sys.argv[1] if len(sys.argv) > 1 else RESULTS_PATH
+ df = load_data(path)
+
+ print("=" * 72)
+ print(" Study 2 Logistic Regression Verification")
+ print("=" * 72)
+
+ print(f"\n Data: {path}")
+ print(f" Total runs: {len(df)}")
+ print(f" Models: {sorted(df['model'].unique())}")
+ print(f" Goal types: {sorted(df['goal_type'].unique())}")
+ print(f" Overall deception rate: {df['deceptive'].sum()}/{len(df)} ({df['deceptive'].mean()*100:.1f}%)")
+
+ print(f"\n Deception rates by goal_type:")
+ for gt in ["bare", "moral", "self_serving"]:
+ sub = df[df["goal_type"] == gt]
+ n_dec = sub["deceptive"].sum()
+ n_tot = len(sub)
+ print(f" {gt:12s}: {n_dec:3d}/{n_tot:3d} ({n_dec/n_tot*100:.1f}%)")
+
+ print(f"\n Deception rates by model:")
+ for model in sorted(df["model"].unique()):
+ sub = df[df["model"] == model]
+ n_dec = sub["deceptive"].sum()
+ n_tot = len(sub)
+ print(f" {model:35s}: {n_dec:3d}/{n_tot:3d} ({n_dec/n_tot*100:.1f}%)")
+
+ section("LOGISTIC REGRESSION: deception ~ model + goal_type")
+ print(" Reference levels: model = deepseek/deepseek-v3.2:nitro, goal_type = self_serving")
+
+ ref_model = "deepseek/deepseek-v3.2:nitro"
+ ref_goal = "self_serving"
+
+ formula = (
+ f"deceptive ~ C(model, Treatment(reference='{ref_model}'))"
+ f" + C(goal_type, Treatment(reference='{ref_goal}'))"
+ )
+ logit = smf.logit(formula, data=df).fit(method="bfgs", maxiter=1000, disp=0)
+
+ print(f"\n Model fit summary:")
+ print(f" Log-Likelihood: {logit.llf:.4f}")
+ print(f" Pseudo R² (McFadden): {logit.prsquared:.4f}")
+ print(f" AIC: {logit.aic:.1f}")
+ print(f" BIC: {logit.bic:.1f}")
+ print(f" Converged: {bool(logit.mle_retvals['converged'])}")
+
+ ## WARNING about perfect separation
+ model_rates = df.groupby("model")["deceptive"].agg(["sum", "count"])
+ model_rates["rate"] = model_rates["sum"] / model_rates["count"]
+ perfect_models = model_rates[model_rates["rate"].isin([0.0, 1.0])]
+ if len(perfect_models) > 0:
+ print(f"\n ⚠ PERFECT SEPARATION DETECTED: {len(perfect_models)} model(s)")
+ for mod, row in perfect_models.iterrows():
+ rate_str = "100.0% deceptive" if row["rate"] == 1.0 else "0.0% deceptive"
+ print(f" {mod}: {rate_str} ({int(row['sum'])}/{int(row['count'])} runs)")
+ print(f" → Coefficients for these models will have very large standard errors.")
+ print(f" → Use Firth's penalized logistic regression for valid inference.")
+
+ print(f"\n {'=' * 72}")
+ print(f" {'Coefficient':50s} {'OR':>8s} {'z':>8s} {'p-value':>10s} {'Sig':>5s}")
+ print(f" {'-' * 50} {'-' * 8} {'-' * 8} {'-' * 10} {'-' * 5}")
+
+ results_rows = []
+ for name in logit.params.index:
+ coef = logit.params[name]
+ pval = logit.pvalues[name]
+ or_val = np.exp(coef)
+ ci = logit.conf_int().loc[name]
+ or_ci = (float(np.exp(ci[0])), float(np.exp(ci[1])))
+ z_val = logit.tvalues[name]
+
+ results_rows.append({
+ "name": name,
+ "coef": coef,
+ "or": or_val,
+ "z": z_val,
+ "p": pval,
+ "ci_low": or_ci[0],
+ "ci_high": or_ci[1],
+ })
+
+ print(f" {name:50s} {or_val:>8.3f} {z_val:>8.3f} {fmt_p(pval):>10s} {sig(pval):>5s}")
+
+ print(f"\n {'OR 95% CIs':}")
+ for r in results_rows:
+ print(f" {r['name']:50s} [{r['ci_low']:.3f}, {r['ci_high']:.3f}]")
+
+ section("COINCIDENCE CHECK: framing coefficient vs p-value")
+ goal_rows = [r for r in results_rows if "goal_type" in r["name"]]
+
+ for r in goal_rows:
+ label = r["name"].replace("C(goal_type, Treatment(reference='self_serving'))[T.", "").rstrip("]")
+ or_val = r["or"]
+ p_val = r["p"]
+ print(f"\n {label} (vs self_serving):")
+ print(f" Coefficient (log-OR) = {r['coef']:.6f}")
+ print(f" OR = {or_val:.6f}")
+ print(f" z = {r['z']:.4f}")
+ print(f" p = {p_val:.6f}")
+ print(f" 95% CI = [{r['ci_low']:.4f}, {r['ci_high']:.4f}]")
+ print()
+
+ or_3dp = round(or_val, 3)
+ p_3dp = round(p_val, 3)
+ print(f" OR (3dp) = {or_3dp}")
+ print(f" p (3dp) = {p_3dp}")
+
+ if or_3dp == p_3dp:
+ print(f"\n *** OR ≈ p at 3dp ({or_3dp}) — this is a COINCIDENCE ***")
+ print()
+ print(" Reasons this is NOT a data error:")
+ print(f" 1. OR = exp(β) = exp({r['coef']:.4f}) = {r['or']:.6f}")
+ print(f" 2. p = 2 × Φ(-|z|) = 2 × Φ(-|{r['z']:.4f}|) = {r['p']:.6f}")
+ print(f" 3. These are computed via entirely different paths:")
+ print(" - OR: simple exponentiation of the coefficient")
+ print(" - p: Wald test (coef / SE) → z → tail probability")
+ print(f" 4. At full precision: OR = {r['or']:.6f} ≠ p = {r['p']:.6f}")
+ print(f" The 3dp match ({or_3dp}) is a rounding coincidence.")
+ else:
+ print(f"\n → OR ({or_3dp}) and p ({p_3dp}) differ at 3dp — no coincidence.")
+
+ section("NESTED MODEL COMPARISON: goal_type effect")
+ null_formula = f"deceptive ~ C(model, Treatment(reference='{ref_model}'))"
+ null_logit = smf.logit(null_formula, data=df).fit(method="bfgs", maxiter=1000, disp=0)
+
+ lr_stat = 2 * (logit.llf - null_logit.llf)
+ lr_df = 2
+ lr_p = chi2.sf(lr_stat, lr_df)
+ print(f"\n Likelihood ratio test: does adding goal_type improve fit?")
+ print(f" Full model LL: {logit.llf:.4f}")
+ print(f" Reduced model LL: {null_logit.llf:.4f}")
+ print(f" χ²({lr_df}) = {lr_stat:.3f}, p = {lr_p:.4f} {sig(lr_p)}")
+ if lr_p >= ALPHA:
+ print(f" → goal_type does NOT significantly improve prediction")
+ print(f" → Consistent with near-identical deception rates across goal types")
+
+ section("REPRODUCING THE OR=0.639/p=0.639 CLAIM")
+ print("""
+ The reported OR=0.639/p=0.639 for moral framing comes from the
+ analyze_study2.py script which uses model_code as an ordinal predictor
+ (pd.Categorical codes: 0,1,2). This is a flawed specification since
+ model is nominal, not ordinal. Replicating that model:""")
+
+ legacy_df = df.copy()
+ legacy_df["bare_code"] = (legacy_df["goal_type"] == "bare").astype(int)
+ legacy_df["moral_code"] = (legacy_df["goal_type"] == "moral").astype(int)
+ legacy_df["model_code"] = pd.Categorical(legacy_df["model"]).codes
+ X_legacy = legacy_df[["bare_code", "moral_code", "model_code"]]
+ X_legacy = sm.add_constant(X_legacy)
+ y_legacy = legacy_df["deceptive"]
+
+ legacy_logit = sm.Logit(y_legacy, X_legacy.astype(float)).fit(disp=0)
+
+ for name, coef, pval in zip(X_legacy.columns, legacy_logit.params, legacy_logit.pvalues):
+ or_val = np.exp(coef)
+ z_val = coef / legacy_logit.bse[name]
+ ci = legacy_logit.conf_int().loc[name]
+ print(f"\n {name:15s}:")
+ print(f" β = {coef:.6f}, SE = {legacy_logit.bse[name]:.6f}")
+ print(f" OR = exp({coef:.6f}) = {or_val:.6f} (→ {or_val:.3f} at 3dp)")
+ print(f" z = {coef:.6f} / {legacy_logit.bse[name]:.6f} = {z_val:.6f}")
+ print(f" p = 2 × Φ(-|{z_val:.6f}|) = {pval:.6f} (→ {pval:.3f} at 3dp)")
+
+ or_3dp = round(or_val, 3)
+ p_3dp = round(pval, 3)
+ if or_3dp == p_3dp:
+ print(f" *** OR = p = {or_3dp} at 3dp — COINCIDENCE confirmed ***")
+
+ section("CONCLUSION")
+ print("""
+ 1. The OR=0.639/p=0.639 match is a NUMERICAL COINCIDENCE, not a data error.
+ - OR = exp(β) and p = 2Φ(-|β/SE|) are mathematically independent.
+ - They only happen to round to the same 3-digit value.
+
+ 2. The original model treats model_code as ordinal (0,1,2), which is
+ inappropriate for nominal model categories.
+
+ 3. Using proper categorical encoding (dummy variables for each model),
+ the moral framing OR = 0.358 with p = 0.493 — no coincidence.
+
+ 4. All goal_type coefficients are non-significant, consistent with
+ the near-identical deception rates across all three conditions.
+
+ 5. Standard MLE struggles with perfect separation (Gemini = 100%
+ deceptive). Firth's penalized regression is recommended for
+ the final dissertation.
+""")
+
+ section("VERIFICATION COMPLETE")
+
+ print(f" Full precision values:")
+ print(f" {'Coefficient':50s} {'Coef':>10s} {'OR':>10s} {'CI low':>10s} "
+ f"{'CI high':>10s} {'z':>8s} {'p':>10s}")
+ print(f" {'-' * 50} {'-' * 10} {'-' * 10} {'-' * 10} {'-' * 10} {'-' * 8} {'-' * 10}")
+ for r in results_rows:
+ name_short = (r["name"][:50] if len(r["name"]) > 50 else r["name"])
+ print(f" {name_short:50s} {r['coef']:>10.4f} {r['or']:>10.4f} "
+ f"{r['ci_low']:>10.4f} {r['ci_high']:>10.4f} "
+ f"{r['z']:>8.3f} {r['p']:>10.6f}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/visualizations.py b/scripts/visualizations.py
new file mode 100644
index 0000000..a5d953e
--- /dev/null
+++ b/scripts/visualizations.py
@@ -0,0 +1,248 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+import os
+
+def setup_theme():
+ sns.set_theme(style="whitegrid")
+ plt.rcParams['font.family'] = 'serif'
+
+# Standard mappings for categories -> (User Friendly Label, Color)
+GB_MAPPING = {
+ 'COMPLIANT_DECEPTION': ('Compliant Deception', '#e74c3c'),
+ 'CONSIDERED_HONEST': ('Considered Honest', '#3498db'),
+ 'ROBUST_HONEST': ('Robust Honest', '#2ecc71'),
+ 'UNINTENTIONAL_FAILURE': ('Unintentional Failure', '#f39c12'),
+ 'FAILED_DECEPTION': ('Failed Deception', '#8e44ad')
+}
+
+BB_MAPPING = {
+ 'MISLEADING': ('Misleading / Framing', '#e74c3c'),
+ 'DECEPTIVE_FRAMING': ('Misleading / Framing', '#e74c3c'),
+ 'ACCURATE': ('Accurate', '#2ecc71'),
+ 'OMITTED': ('Omitted Critical', '#f39c12'),
+ 'OMITTED_CRITICAL': ('Omitted Critical', '#f39c12'),
+ 'FABRICATED': ('Fabricated', '#c0392b')
+}
+
+def plot_composition(df, column, mapping, title_prefix, filename, expected_order):
+ total = len(df)
+ counts = df[column].value_counts()
+
+ percentages = []
+ colors = []
+ labels = []
+
+ for cat in expected_order:
+ count = counts.get(cat, 0)
+ if count > 0:
+ pct = (count / total) * 100
+ percentages.append(pct)
+ labels.append(mapping[cat][0])
+ colors.append(mapping[cat][1])
+
+ plt.figure(figsize=(10, 2))
+ left = 0
+ for i in range(len(percentages)):
+ pct = percentages[i]
+ plt.barh(0, pct, left=left, color=colors[i], label=labels[i], height=0.5)
+ if pct > 5:
+ # White text, unless it's a light color like yellow/orange
+ text_color = 'black' if colors[i] == '#f39c12' else 'white'
+ plt.text(left + pct/2, 0, f"{pct:.1f}%", ha='center', va='center', fontweight='bold', color=text_color)
+ left += pct
+
+ plt.axis('off')
+ plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.2), ncol=3, frameon=False, fontsize=11)
+ plt.tight_layout()
+ os.makedirs('docs/final/images', exist_ok=True)
+ plt.savefig(f'docs/final/images/{filename}', dpi=300, bbox_inches='tight')
+ plt.close()
+
+def generate_composition_charts():
+ # --- V1 ---
+ df_v1 = pd.read_csv('logs/full_experiment/results.csv')
+ gb_order = ['COMPLIANT_DECEPTION', 'CONSIDERED_HONEST', 'ROBUST_HONEST', 'UNINTENTIONAL_FAILURE', 'FAILED_DECEPTION']
+ bb_order = ['MISLEADING', 'DECEPTIVE_FRAMING', 'ACCURATE', 'OMITTED', 'OMITTED_CRITICAL', 'FABRICATED']
+
+ plot_composition(df_v1, 'glassbox_category', GB_MAPPING, 'Glassbox', 'v1_glassbox_composition.png', gb_order)
+ plot_composition(df_v1, 'blackbox_category', BB_MAPPING, 'Blackbox', 'v1_blackbox_composition.png', bb_order)
+
+ # --- V2 ---
+ df_study1 = pd.read_csv('logs/v2_study1/study1_results.csv')
+ df_study2 = pd.read_csv('logs/v2_study2/study2_results.csv')
+ df_v2 = pd.concat([df_study1, df_study2], ignore_index=True)
+
+ plot_composition(df_v2, 'glassbox_category', GB_MAPPING, 'Glassbox', 'v2_glassbox_composition.png', gb_order)
+ plot_composition(df_v2, 'blackbox_category', BB_MAPPING, 'Blackbox', 'v2_blackbox_composition.png', bb_order)
+
+def generate_v1_frontier_stealth_gap():
+ df = pd.read_csv('logs/full_experiment/results.csv')
+ frontier_models = ['gemini', 'deepseek', 'kimi', 'qwen']
+ df_frontier = df[df['model'].str.lower().str.contains('|'.join(frontier_models))]
+
+ oversight_order = ['low', 'mid', 'high']
+ gb_deception = df_frontier.groupby('oversight')['glassbox_category'].apply(lambda x: (x == 'COMPLIANT_DECEPTION').mean() * 100).reindex(oversight_order)
+ bb_deception = df_frontier.groupby('oversight')['blackbox_category'].apply(lambda x: (x != 'ACCURATE').mean() * 100).reindex(oversight_order)
+
+ plt.figure(figsize=(10, 6))
+ plt.plot(gb_deception.index, gb_deception.values, marker='s', linewidth=3, markersize=10, color='#2c3e50', label='Glassbox')
+ plt.plot(bb_deception.index, bb_deception.values, marker='o', linewidth=3, markersize=10, color='#e74c3c', label='Blackbox')
+ plt.fill_between(oversight_order, bb_deception.values, gb_deception.values, color='gray', alpha=0.15, label='Stealth Gap')
+
+ plt.xlabel('Oversight Level', fontsize=13)
+ plt.ylabel('Deception Rate (%)', fontsize=13)
+ plt.ylim(0, 105)
+ plt.legend(fontsize=11, loc='lower left')
+
+ for i, level in enumerate(oversight_order):
+ gap = gb_deception[level] - bb_deception[level]
+ plt.text(i, (gb_deception[level] + bb_deception[level])/2, f'Gap: {gap:.1f}%',
+ ha='center', va='center', fontweight='bold', color='#2c3e50',
+ bbox=dict(facecolor='white', alpha=0.8, edgecolor='none', boxstyle='round,pad=0.2'))
+
+ plt.grid(True, linestyle='--', alpha=0.7)
+ plt.tight_layout()
+ plt.savefig('docs/final/images/v1_frontier_stealth_gap.png', dpi=300)
+ plt.close()
+
+def generate_study1_deception_rates():
+ df = pd.read_csv('logs/v2_study1/study1_results.csv')
+ oversight_order = ['low', 'mid', 'high']
+ gb_deception = df.groupby('oversight')['glassbox_category'].apply(lambda x: (x == 'COMPLIANT_DECEPTION').mean() * 100).reindex(oversight_order)
+ bb_deception = df.groupby('oversight')['blackbox_category'].apply(lambda x: (x != 'ACCURATE').mean() * 100).reindex(oversight_order)
+
+ plt.figure(figsize=(10, 6))
+ plt.plot(gb_deception.index, gb_deception.values, marker='s', linewidth=3, markersize=10, color='#2c3e50', label='Glassbox')
+ plt.plot(bb_deception.index, bb_deception.values, marker='o', linewidth=3, markersize=10, color='#e74c3c', label='Blackbox')
+ plt.fill_between(oversight_order, bb_deception.values, gb_deception.values, color='gray', alpha=0.2, label='Stealth Gap')
+
+ plt.xlabel('Oversight Level (Audit Rate)', fontsize=13)
+ plt.ylabel('Deception Rate (%)', fontsize=13)
+ plt.ylim(0, 100)
+ plt.legend(fontsize=11, loc='upper left')
+
+ for i, level in enumerate(oversight_order):
+ gap = gb_deception[level] - bb_deception[level]
+ plt.text(i, (gb_deception[level] + bb_deception[level])/2, f'Gap: {gap:.1f}%',
+ ha='center', va='center', fontweight='bold', color='#2980b9',
+ bbox=dict(facecolor='white', alpha=0.8, edgecolor='none', boxstyle='round,pad=0.2'))
+
+ plt.grid(True, linestyle='--', alpha=0.7)
+ plt.tight_layout()
+ plt.savefig('docs/final/images/study1_deception_rates.png', dpi=300)
+ plt.close()
+
+def generate_study1_deception_rates_grouped():
+ df = pd.read_csv('logs/v2_study1/study1_results.csv')
+ def calc_rates(group):
+ total = len(group)
+ gb_deception = sum(group['glassbox_category'] == 'COMPLIANT_DECEPTION')
+ bb_deception = sum(group['blackbox_category'].isin(['DECEPTIVE_FRAMING', 'OMITTED_CRITICAL', 'FABRICATED']))
+ return pd.Series({
+ 'Glassbox': (gb_deception / total) * 100 if total > 0 else 0,
+ 'Blackbox': (bb_deception / total) * 100 if total > 0 else 0
+ })
+ rates = df.groupby('oversight').apply(calc_rates).reindex(['low', 'mid', 'high'])
+
+ fig, ax = plt.subplots(figsize=(8, 6))
+ x = np.arange(len(rates.index))
+ width = 0.35
+
+ rects1 = ax.bar(x - width/2, rates['Glassbox'], width, label='Glassbox', color='#2c3e50', edgecolor='white', zorder=3)
+ rects2 = ax.bar(x + width/2, rates['Blackbox'], width, label='Blackbox', color='#e74c3c', edgecolor='white', zorder=3)
+
+ ax.set_ylabel('Deception Rate (%)', fontsize=12)
+ ax.set_xlabel('Oversight Level', fontsize=12)
+ ax.set_xticks(x)
+ ax.set_xticklabels(['Low', 'Medium', 'High'], fontsize=11)
+ ax.set_ylim(0, 105)
+ ax.yaxis.grid(True, linestyle='--', alpha=0.7, zorder=0)
+ ax.legend(fontsize=11)
+
+ def autolabel(rects):
+ for rect in rects:
+ height = rect.get_height()
+ ax.annotate(f'{height:.1f}%',
+ xy=(rect.get_x() + rect.get_width() / 2, height),
+ xytext=(0, 5), textcoords="offset points",
+ ha='center', va='bottom', fontsize=10, fontweight='bold')
+ autolabel(rects1)
+ autolabel(rects2)
+
+ plt.tight_layout()
+ plt.savefig('docs/final/images/study1_deception_rates_grouped.png', dpi=300, bbox_inches='tight')
+ plt.close()
+
+def generate_study2_framing_bars():
+ df = pd.read_csv('logs/v2_study2/study2_results.csv')
+
+ # Extract framing from run_id
+ df['framing'] = df['run_id'].apply(lambda x: x.split('/')[2])
+
+ framing_order = ['bare', 'self_serving', 'moral']
+ framing_labels = ['Bare', 'Self-Serving', 'Moral']
+
+ def calc_rates(group):
+ total = len(group)
+ gb_deception = sum(group['glassbox_category'] == 'COMPLIANT_DECEPTION')
+ bb_deception = sum(group['blackbox_category'].isin(['DECEPTIVE_FRAMING', 'OMITTED_CRITICAL', 'FABRICATED']))
+ return pd.Series({
+ 'Glassbox': (gb_deception / total) * 100 if total > 0 else 0,
+ 'Blackbox': (bb_deception / total) * 100 if total > 0 else 0
+ })
+
+ rates = df.groupby('framing').apply(calc_rates).reindex(framing_order)
+
+ fig, ax = plt.subplots(figsize=(8, 6))
+ x = np.arange(len(rates.index))
+ width = 0.35
+
+ # Colors consistent with Study 1 Grouped Chart
+ rects1 = ax.bar(x - width/2, rates['Glassbox'], width, label='Glassbox', color='#2c3e50', edgecolor='white', zorder=3)
+ rects2 = ax.bar(x + width/2, rates['Blackbox'], width, label='Blackbox', color='#e74c3c', edgecolor='white', zorder=3)
+
+ # Add horizontal trend lines for both
+ gb_mean = rates['Glassbox'].mean()
+ bb_mean = rates['Blackbox'].mean()
+
+ ax.axhline(gb_mean, color='#2c3e50', linestyle='--', linewidth=1.5, alpha=0.5, zorder=2)
+ ax.axhline(bb_mean, color='#e74c3c', linestyle='--', linewidth=1.5, alpha=0.5, zorder=2)
+
+ ax.set_ylabel('Deception Rate (%)', fontsize=12)
+ ax.set_xlabel('Hidden Goal Framing', fontsize=12)
+ ax.set_xticks(x)
+ ax.set_xticklabels(framing_labels, fontsize=11)
+ ax.set_ylim(0, 105)
+ ax.yaxis.grid(True, linestyle='--', alpha=0.7, zorder=0)
+ ax.legend(fontsize=11, loc='lower right')
+
+ def autolabel(rects):
+ for rect in rects:
+ height = rect.get_height()
+ ax.annotate(f'{height:.1f}%',
+ xy=(rect.get_x() + rect.get_width() / 2, height),
+ xytext=(0, 5), textcoords="offset points",
+ ha='center', va='bottom', fontsize=10, fontweight='bold')
+
+ autolabel(rects1)
+ autolabel(rects2)
+
+ plt.tight_layout()
+ plt.savefig('docs/final/images/study2_framing_bars.png', dpi=300, bbox_inches='tight')
+ plt.close()
+
+if __name__ == "__main__":
+ setup_theme()
+ print("Generating v1 and v2 composition charts dynamically...")
+ generate_composition_charts()
+ print("Generating v1_frontier_stealth_gap.png...")
+ generate_v1_frontier_stealth_gap()
+ print("Generating study1_deception_rates.png...")
+ generate_study1_deception_rates()
+ print("Generating study1_deception_rates_grouped.png...")
+ generate_study1_deception_rates_grouped()
+ print("Generating study2_framing_bars.png...")
+ generate_study2_framing_bars()
+ print("All final visualizations generated successfully directly into docs/final/images/.")