blob: 637364c0aa28c875f5f1d792068f787a3c3bc7c1 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
|
# Experiment Configuration
# Define providers, models, scenarios, and runs
# Default parameters (can be overridden per model)
defaults:
temperature: 1.0
# max_tokens: 4096
oversight: high
include_reasoning_in_context: true
# Provider configurations
providers:
groq:
api_key_env: GROQ_API_KEY
base_url: https://api.groq.com/openai/v1
openai:
api_key_env: OPENAI_API_KEY
base_url: https://api.openai.com/v1
google:
api_key_env: GOOGLE_API_KEY
base_url: https://generativelanguage.googleapis.com/v1beta/openai/
openrouter:
api_key_env: OPENROUTER_API_KEY
base_url: https://openrouter.ai/api/v1
anthropic:
api_key_env: ANTHROPIC_API_KEY
moonshot:
api_key_env: OPENROUTER_API_KEY
base_url: https://openrouter.ai/api/v1
# Models to test
models:
- id: openai/gpt-oss-20b
provider: groq
reasoning_effort: medium
# - id: z-ai/glm-5
# provider: openrouter
# extra_body:
# thinking:
# enabled: true
# - id: gpt-4.5
# provider: openai
# temperature: 0.7
# - id: claude-haiku-4-5
# provider: anthropic
# temperature: 0.7
# extra_body:
# thinking:
# type: "enabled"
# budget_tokens: 10000
# - id: gemini-3-flash-preview
# provider: google
# temperature: 1
# extra_body:
# extra_body:
# google:
# thinking_config:
# include_thoughts: True
# - id: anthropic/claude-sonnet-4-5
# provider: openrouter
# temperature: 0.7
# - id: moonshotai/kimi-k2.5
# provider: moonshot
# temperature: 1.0
# extra_body:
# reasoning:
# enabled: True
# Scenarios to run
scenarios:
# - path: scenarios/child_protection
# runs: 1
- path: scenarios/corporate_sabotage_v2
runs: 1
# - path: scenarios/log_redaction
# runs: 3
# Oversight levels to test
oversight_levels:
- low
# - medium
- high
# Output settings
output:
dir: logs
# save_vfs_state: false
# Judge configuration
judge:
model: claude-sonnet-4-20250514
temperature: 0
log_dir: judge_logs
|
