diff options
| -rw-r--r-- | .env.example | 17 | ||||
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | config.yaml.example (renamed from config.yaml) | 60 | ||||
| -rw-r--r-- | src/judge.py | 7 |
4 files changed, 39 insertions, 48 deletions
diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..fcb5823 --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# API Keys for Experiment Providers +# Copy this file to .env and fill in your keys. + +# OpenAI +# OPENAI_API_KEY=sk-... + +# Google (Gemini) +# GOOGLE_API_KEY=... + +# OpenRouter +# OPENROUTER_API_KEY=sk-or-v1-... + +# Anthropic (required for judging) +# ANTHROPIC_API_KEY=sk-ant-... + +# Groq +# GROQ_API_KEY=gsk_... @@ -6,9 +6,12 @@ __pycache__/ *.pyc .env +config.yaml # Generated data output/ interrogation_logs/ +judge_logs/ tests/__pycache__/ src/__pycache__/ +.aider* diff --git a/config.yaml b/config.yaml.example index 637364c..4b6f609 100644 --- a/config.yaml +++ b/config.yaml.example @@ -1,5 +1,6 @@ # Experiment Configuration -# Define providers, models, scenarios, and runs +# Copy this file to config.yaml and customise for your runs. +# Define providers, models, scenarios, and runs. # Default parameters (can be overridden per model) defaults: @@ -28,63 +29,26 @@ providers: api_key_env: OPENROUTER_API_KEY base_url: https://openrouter.ai/api/v1 - # Models to test models: - - id: openai/gpt-oss-20b - provider: groq - reasoning_effort: medium - - - -# - id: z-ai/glm-5 -# provider: openrouter -# extra_body: -# thinking: -# enabled: true - -# - id: gpt-4.5 -# provider: openai -# temperature: 0.7 + - id: gemini-3-flash-preview + provider: google + temperature: 1 -# - id: claude-haiku-4-5 -# provider: anthropic -# temperature: 0.7 -# extra_body: -# thinking: -# type: "enabled" -# budget_tokens: 10000 + # - id: openai/gpt-oss-20b + # provider: groq -# - id: gemini-3-flash-preview -# provider: google -# temperature: 1 -# extra_body: -# extra_body: -# google: -# thinking_config: -# include_thoughts: True - -# - id: anthropic/claude-sonnet-4-5 -# provider: openrouter -# temperature: 0.7 - -# - id: moonshotai/kimi-k2.5 -# provider: moonshot -# temperature: 1.0 -# extra_body: -# reasoning: -# enabled: True + # - id: claude-haiku-4-5 + # provider: anthropic + # temperature: 0.7 # Scenarios to run scenarios: -# - path: scenarios/child_protection -# runs: 1 - - path: scenarios/corporate_sabotage_v2 runs: 1 -# - path: scenarios/log_redaction -# runs: 3 + # - path: scenarios/child_protection + # runs: 1 # Oversight levels to test oversight_levels: diff --git a/src/judge.py b/src/judge.py index 0d5117d..076b04a 100644 --- a/src/judge.py +++ b/src/judge.py @@ -13,6 +13,13 @@ from typing import Any, Dict, List, Optional import anthropic +# Load environment variables from .env file +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + pass # python-dotenv not installed + # --------------------------------------------------------------------------- # Constants |