blob: 7253c199c11436240b138026f5895010fdf193c4 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
|
# Experiment Configuration
# Copy this file to config.yaml and customise for your runs.
# Define providers, models, scenarios, and runs.
# Default parameters (can be overridden per model)
defaults:
temperature: 1.0
# max_tokens: 4096
oversight: high
include_reasoning_in_context: true
generate_baseline: true # Set to false to skip baseline generation
# Provider configurations
providers:
groq:
api_key_env: GROQ_API_KEY
base_url: https://api.groq.com/openai/v1
openai:
api_key_env: OPENAI_API_KEY
base_url: https://api.openai.com/v1
google:
api_key_env: GOOGLE_API_KEY
base_url: https://generativelanguage.googleapis.com/v1beta/openai/
openrouter:
api_key_env: OPENROUTER_API_KEY
base_url: https://openrouter.ai/api/v1
anthropic:
api_key_env: ANTHROPIC_API_KEY
moonshot:
api_key_env: OPENROUTER_API_KEY
base_url: https://openrouter.ai/api/v1
# Models to test (frontier models with reasoning traces, via OpenRouter)
models:
- id: google/gemini-3.1-pro-preview
provider: openrouter
extra_body:
reasoning:
effort: "high"
exclude: false
- id: z-ai/glm-5
provider: openrouter
extra_body:
reasoning:
effort: "high"
exclude: false
- id: deepseek/deepseek-v3.2
provider: openrouter
extra_body:
reasoning:
effort: "high"
exclude: false
# Scenarios to run
scenarios:
- path: scenarios/corporate_sabotage_v2
runs: 20
- path: scenarios/child_protection
runs: 20
- path: scenarios/pharma_trial
runs: 20
# Oversight levels to test
oversight_levels:
- low
- high
# Output settings
output:
dir: logs
# save_vfs_state: false
# Judge configuration
judge:
model: claude-sonnet-4-20250514
temperature: 0
log_dir: judge_logs
# Logging configuration
logging:
level: 3 # 1-4, controls verbosity (1=CRITICAL, 2=WARNING, 3=INFO, 4=DEBUG)
format: "[{level}] {message}"
output: console # console, file, or both (auto-set to 'both' at level 4)
file: logs/experiment.log
|
