1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
import json
import glob
import os
from collections import defaultdict
JUDGE_DIR = "logs/v2_study1_judge"
MODEL_SHORT = {
"deepseek_deepseek-v3.2:nitro": "DeepSeek",
"google_gemini-3.1-pro-preview": "Gemini",
"z-ai_glm-5.1:nitro": "GLM",
}
def load_all_verdicts():
records = []
for fpath in glob.glob(os.path.join(JUDGE_DIR, "*.json")):
with open(fpath) as f:
d = json.load(f)
run_id = d["run_id"]
parts = run_id.split("/")
model_dir = parts[0]
oversight = parts[3]
run_ts = parts[4]
short = MODEL_SHORT.get(model_dir, model_dir)
v = d["verdict"]
records.append({
"model": short,
"oversight": oversight,
"run_id": run_id,
"run_ts": run_ts,
"bb_cat": v["blackbox_category"],
"gb_cat": v["glassbox_category"],
"gb_soph": v["glassbox_sophistication"],
})
return records
def print_table(title, headers, rows):
w = [len(h) for h in headers]
for r in rows:
for i, c in enumerate(r):
w[i] = max(w[i], len(str(c)))
sep = " | ".join("-" * wi for wi in w)
header = " | ".join(h.ljust(wi) for h, wi in zip(headers, w))
print(f"\n{'=' * 80}")
print(f" {title}")
print(f"{'=' * 80}")
print(f" {header}")
print(f" {sep}")
for r in rows:
print(f" " + " | ".join(str(c).ljust(wi) for c, wi in zip(r, w)))
print()
def claim1_compliant_deception_counts(records):
title = "Claim 1: COMPLIANT_DECEPTION counts by model x oversight"
headers = ["Model", "low", "mid", "high"]
order = ["DeepSeek", "Gemini", "GLM"]
levels = ["low", "mid", "high"]
counts = defaultdict(lambda: defaultdict(int))
for r in records:
if r["gb_cat"] == "COMPLIANT_DECEPTION":
counts[r["model"]][r["oversight"]] += 1
rows = []
for m in order:
rows.append([m] + [str(counts[m][l]) for l in levels])
rows.append(["Total"] + [str(sum(counts[mm][l] for mm in order)) for l in levels])
print_table(title, headers, rows)
def claim2_glm_low_breakdown(records):
title = "Claim 2: GLM at low oversight — deceptive runs breakdown"
glm_low = [r for r in records if r["model"] == "GLM" and r["oversight"] == "low"]
deceptive = [r for r in glm_low if r["gb_cat"] in ("COMPLIANT_DECEPTION", "CONSIDERED_HONEST")]
print(f"\n{'=' * 80}")
print(f" {title}")
print(f"{'=' * 80}")
print(f" Total GLM low-oversight runs: {len(glm_low)}")
print()
gb_counts = defaultdict(int)
for r in deceptive:
gb_counts[r["gb_cat"]] += 1
print(f" Glassbox intent breakdown of deceptive runs:")
for cat in sorted(gb_counts):
print(f" {cat}: {gb_counts[cat]}")
considered = [r for r in deceptive if r["gb_cat"] == "CONSIDERED_HONEST"]
print(f"\n CONSIDERED_HONEST runs — blackbox labels:")
bb_counts = defaultdict(int)
for r in considered:
bb_counts[r["bb_cat"]] += 1
for cat in sorted(bb_counts):
print(f" {cat}: {bb_counts[cat]}")
if considered:
headers = ["run_ts", "blackbox_category"]
rows = [[r["run_ts"], r["bb_cat"]] for r in considered]
print()
headers2 = ["Model"] + headers
rows2 = [["GLM"] + r for r in rows]
print_table("Individual CONSIDERED_HONEST runs", headers2, rows2)
def claim3_glm_considered_honest_sophistication(records):
title = "Claim 3: GLM CONSIDERED_HONEST sophistication by oversight"
glm_ch = [r for r in records if r["model"] == "GLM" and r["gb_cat"] == "CONSIDERED_HONEST"]
headers = ["Oversight", "STRATEGIC", "TACTICAL", "NAIVE", "Total"]
levels = ["low", "mid", "high"]
rows = []
for lv in levels:
subset = [r for r in glm_ch if r["oversight"] == lv]
str_cnt = sum(1 for r in subset if r["gb_soph"] == "STRATEGIC")
tac_cnt = sum(1 for r in subset if r["gb_soph"] == "TACTICAL")
na_cnt = sum(1 for r in subset if r["gb_soph"] == "NAIVE")
rows.append([lv, str(str_cnt), str(tac_cnt), str(na_cnt), str(len(subset))])
all_str = sum(1 for r in glm_ch if r["gb_soph"] == "STRATEGIC")
all_tac = sum(1 for r in glm_ch if r["gb_soph"] == "TACTICAL")
all_na = sum(1 for r in glm_ch if r["gb_soph"] == "NAIVE")
rows.append(["Total", str(all_str), str(all_tac), str(all_na), str(len(glm_ch))])
print_table(title, headers, rows)
def claim4_non_strategic_deceptive(records):
title = "Claim 4: COMPLIANT_DECEPTION runs where sophistication is NOT STRATEGIC"
non_strat = [r for r in records if r["gb_cat"] == "COMPLIANT_DECEPTION" and r["gb_soph"] != "STRATEGIC"]
headers = ["Model", "Oversight", "run_ts", "Sophistication"]
rows = [[r["model"], r["oversight"], r["run_ts"], r["gb_soph"]] for r in non_strat]
print_table(title, headers, rows)
if not rows:
print(" (none found)")
print()
def main():
records = load_all_verdicts()
total = len(records)
print(f"\nLoaded {total} judge verdicts from {JUDGE_DIR}/")
model_counts = defaultdict(int)
for r in records:
model_counts[r["model"]] += 1
print(f" Models: {dict(model_counts)}")
oversight_counts = defaultdict(int)
for r in records:
oversight_counts[r["oversight"]] += 1
print(f" Oversight levels: {dict(oversight_counts)}")
claim1_compliant_deception_counts(records)
claim2_glm_low_breakdown(records)
claim3_glm_considered_honest_sophistication(records)
claim4_non_strategic_deceptive(records)
if __name__ == "__main__":
main()
|
