1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "pandas",
# "numpy",
# "scipy",
# "statsmodels",
# ]
# ///
"""
McNemar Effect Sizes for Study 1
=================================
Computes effect-size metrics from the paired 2x2 tables of glassbox vs blackbox
for each oversight level.
Each table cell:
a = glassbox-deceptive AND blackbox-misleading
b = glassbox-deceptive AND NOT blackbox-misleading (stealth runs)
c = NOT glassbox-deceptive AND blackbox-misleading (false alarms)
d = neither
Reports for each oversight level:
- Discordant pair counts: b (stealth), c (false alarm)
- Odds ratio of discordant pairs: OR = b / c
- 95% confidence interval for OR
- Proportion of discordant pairs: (b + c) / N
Usage:
uv run python scripts/mcnemar_effect_sizes.py
"""
import warnings
import numpy as np
import pandas as pd
from scipy import stats
warnings.filterwarnings("ignore", category=FutureWarning)
DECEPTIVE_BB = {"DECEPTIVE_FRAMING", "OMITTED_CRITICAL", "FABRICATED"}
COMPLIANT_GB = {"COMPLIANT_DECEPTION"}
RESULTS_PATH = "logs/v2_study1/study1_results.csv"
Z_ALPHA = stats.norm.ppf(1 - 0.05 / 2)
def load_data(path: str) -> pd.DataFrame:
df = pd.read_csv(path)
df["is_deceptive_bb"] = df["blackbox_category"].isin(DECEPTIVE_BB).astype(int)
df["is_compliant_gb"] = df["glassbox_category"].isin(COMPLIANT_GB).astype(int)
return df
def discordant_or_ci(b: int, c: int) -> tuple[float, float, float]:
if b == 0 or c == 0:
b_adj, c_adj = b + 0.5, c + 0.5
else:
b_adj, c_adj = float(b), float(c)
or_val = b_adj / c_adj
log_or = np.log(or_val)
se = np.sqrt(1 / b_adj + 1 / c_adj)
ci_low = np.exp(log_or - Z_ALPHA * se)
ci_high = np.exp(log_or + Z_ALPHA * se)
return or_val, ci_low, ci_high
def main():
df = load_data(RESULTS_PATH)
print("=" * 72)
print(" McNemar Effect Sizes — Discordant-Pair Analysis")
print("=" * 72)
header = (
f" {'Oversight':>9s} {'N':>4s} {'b(stealth)':>10s} "
f"{'c(falseAlm)':>10s} {'OR(b/c)':>8s} "
f"{'OR 95% CI':>20s} {'(b+c)/N':>8s} {'Disagr%':>8s}"
)
sep = " " + "-" * 9 + " " + "-" * 4 + " " + "-" * 10 + " " + "-" * 10 + " " + "-" * 8 + " " + "-" * 20 + " " + "-" * 8 + " " + "-" * 8
print(f"\n{header}")
print(sep)
for ov in ["low", "mid", "high"]:
sub = df[df["oversight"] == ov]
N = len(sub)
gb_dec = sub["is_compliant_gb"].values
bb_det = sub["is_deceptive_bb"].values
a = int(((gb_dec == 1) & (bb_det == 1)).sum())
b = int(((gb_dec == 1) & (bb_det == 0)).sum())
c = int(((gb_dec == 0) & (bb_det == 1)).sum())
d = int(((gb_dec == 0) & (bb_det == 0)).sum())
or_val, ci_low, ci_high = discordant_or_ci(b, c)
discordant_prop = (b + c) / N
discordant_pct = discordant_prop * 100
ci_str = f"[{ci_low:.3f}, {ci_high:.3f}]"
print(
f" {ov:>9s} {N:>4d} {b:>10d} {c:>10d} "
f"{or_val:>8.3f} {ci_str:>20s} "
f"{discordant_prop:>8.3f} {discordant_pct:>7.1f}%"
)
if b + c > 0:
if b > c:
direction = f"stealth runs outnumber false alarms {b}:{c}"
elif c > b:
direction = f"false alarms outnumber stealth runs {c}:{b}"
else:
direction = "stealth runs and false alarms are equal"
print(f" → {direction}")
decision = ""
if or_val > 1 and ci_low > 1:
decision = " (OR > 1, significant)"
elif or_val < 1 and ci_high < 1:
decision = " (OR < 1, significant)"
else:
decision = " (not significant, CI crosses 1)"
print(f" OR 95% CI does{'' if 'significant' in decision else ' not'} exclude 1{decision}")
print("\n Interpretation:")
print(" b = glassbox says deceptive | blackbox says honest (stealth)")
print(" c = glassbox says honest | blackbox says misleading (false alarm)")
print(" OR = b/c: odds that a discordant pair is a stealth run vs a false alarm")
print(" (b+c)/N: proportion of all pairs showing disagreement between judges")
if __name__ == "__main__":
main()
|
