From f19aa5de3e3f1c3c0b9ce90913588b9e956d4156 Mon Sep 17 00:00:00 2001 From: CaptainJack2491 Date: Mon, 9 Mar 2026 20:35:34 +0000 Subject: feat: add total token usage chart and update documentation This commit: - Adds a 'Total Token Usage by Model' chart alongside the average token chart - Updates the web_gui_plan.md to reflect the removal of execution APIs and the addition of the new data visualization features. --- api/static/js/components/results.js | 59 ++++++++++++++++++++++++++++++++++--- docs/web_gui_plan.md | 56 +++++++++++------------------------ 2 files changed, 72 insertions(+), 43 deletions(-) diff --git a/api/static/js/components/results.js b/api/static/js/components/results.js index 9b530f4..b143fa3 100644 --- a/api/static/js/components/results.js +++ b/api/static/js/components/results.js @@ -231,7 +231,8 @@ export const results = { } if (cols.includes('model') && cols.includes('total_tokens')) { - this.renderTokenUsageChart('Average Token Usage by Model', data); + this.renderAverageTokenUsageChart('Average Token Usage by Model', data); + this.renderTotalTokenUsageChart('Total Token Usage by Model', data); } if (ui.elements.interactiveCharts.innerHTML === '') { @@ -239,7 +240,7 @@ export const results = { } }, - renderTokenUsageChart(title, data) { + renderAverageTokenUsageChart(title, data) { const metrics = {}; // { model: { total: 0, count: 0, prompt: 0, completion: 0 } } data.forEach(row => { @@ -260,12 +261,12 @@ export const results = { const datasets = [ { - label: 'Prompt Tokens', + label: 'Avg Prompt Tokens', data: labels.map(l => metrics[l].prompt / metrics[l].count), backgroundColor: '#3b82f6', }, { - label: 'Completion Tokens', + label: 'Avg Completion Tokens', data: labels.map(l => metrics[l].completion / metrics[l].count), backgroundColor: '#10b981', } @@ -291,6 +292,56 @@ export const results = { activeCharts.push(chart); }, + renderTotalTokenUsageChart(title, data) { + const metrics = {}; // { model: { prompt: 0, completion: 0 } } + + data.forEach(row => { + const model = row.model || 'Unknown'; + if (!row.total_tokens) return; + + if (!metrics[model]) { + metrics[model] = { prompt: 0, completion: 0 }; + } + metrics[model].prompt += row.prompt_tokens || 0; + metrics[model].completion += row.completion_tokens || 0; + }); + + const labels = Object.keys(metrics).sort(); + if (labels.length === 0) return; + + const datasets = [ + { + label: 'Total Prompt Tokens', + data: labels.map(l => metrics[l].prompt), + backgroundColor: '#6366f1', + }, + { + label: 'Total Completion Tokens', + data: labels.map(l => metrics[l].completion), + backgroundColor: '#f59e0b', + } + ]; + + const config = { + type: 'bar', + data: { labels, datasets }, + options: { + ...this.getChartOptions(title), + scales: { + x: { stacked: true, ticks: { color: '#9ea0a6' }, grid: { color: '#2a2a2e' } }, + y: { stacked: true, ticks: { color: '#9ea0a6' }, grid: { color: '#2a2a2e' } } + } + } + }; + + const canvasId = `chart-${Math.random().toString(36).substr(2, 9)}`; + this.createChartContainer(canvasId, config); + + const ctx = document.getElementById(canvasId).getContext('2d'); + const chart = new Chart(ctx, config); + activeCharts.push(chart); + }, + renderDoughnutChart(title, data, column) { const counts = {}; data.forEach(row => { diff --git a/docs/web_gui_plan.md b/docs/web_gui_plan.md index 3b041ff..f43b0d0 100644 --- a/docs/web_gui_plan.md +++ b/docs/web_gui_plan.md @@ -79,33 +79,11 @@ dissertation/ ## API Endpoints -### 1. Configuration +### 1. Configuration & Discovery (Removed) -| Method | Endpoint | Description | -|--------|----------|-------------| -| GET | `/api/config` | Read current `config.yaml` | -| PUT | `/api/config` | Update/save `config.yaml` | -| GET | `/api/logging` | Get current logging configuration (file path, level) | - -### 2. Discovery - -| Method | Endpoint | Description | -|--------|----------|-------------| -| GET | `/api/scenarios` | List all available scenarios from `scenarios/` | -| GET | `/api/scenarios/{name}` | Get details (oversight levels, files) for a scenario | -| GET | `/api/models` | List models from config | -| GET | `/api/providers` | List providers from config | - -### 3. Execution - -| Method | Endpoint | Description | -|--------|----------|-------------| -| POST | `/api/run` | Start an experiment run (non-blocking) | -| GET | `/api/run/status` | Get current run status (idle/running/complete) | -| DELETE | `/api/run` | Cancel current run | -| GET | `/api/logs/stream` | Server-Sent Events (SSE) for live log streaming | +*Note: In v0.2.0, the GUI was pivoted to a pure Data Visualization Dashboard. Execution logic (triggering runs, stopping runs, tailing live logs, and configuration editing) has been removed to keep the interface focused on analysis.* -### 4. Results & Analysis +### 2. Results & Analysis | Method | Endpoint | Description | |--------|----------|-------------| @@ -124,25 +102,23 @@ dissertation/ 1. **Setup**: Add `fastapi`, `uvicorn` to dependencies (via `uv add`) 2. **Server**: Create `api/server.py` with basic FastAPI app -3. **Config Endpoint**: Wrap `ConfigLoader` to expose config read/write -4. **Scenario Discovery**: Auto-scan `scenarios/` directory -5. **Run Trigger**: Execute `uv run src/main.py` as subprocess -6. **Log Streaming**: Implement SSE endpoint to tail log file -7. **Results Endpoint**: Read CSV/JSON files dynamically based on URL queries. +3. **Data Serving**: Read CSV/JSON files dynamically based on URL queries. +4. **Token Usage**: The CSV data includes token metrics that were injected during the run or post-processed. ### Phase 2: Frontend (Vanilla ES6) 1. **Static Architecture**: Implement a modular JS structure (`js/components/`). -2. **Dashboard Layout**: Fixed left sidebar for configuration, main viewport for data analysis, and a docked live terminal. -3. **Aesthetics**: "Industrial Control Room" dark theme (high contrast, sans-serif UI, monospace terminal). -4. **Live Data**: Connect `EventSource` to SSE endpoint for a real-time log feed. -5. **Interactive Visualizations**: Integrate `Chart.js` to dynamically generate pie and bar charts from CSV data endpoints. +2. **Dashboard Layout**: Fixed left sidebar for global filtering, main viewport for data analysis. +3. **Aesthetics**: "Industrial Control Room" dark theme (high contrast, sans-serif UI). +4. **Interactive Visualizations**: Integrate `Chart.js` to dynamically generate pie, bar, and stacked bar charts from CSV data endpoints. +5. **Chart Zooming**: Implement a full-screen modal overlay for examining charts closely. ### Phase 3: Integration & Polish 1. **Auto-discovery**: Ensure new scenarios and result files populate dropdowns automatically. 2. **Data Parsing**: Ensure long text fields (like judge justifications) wrap nicely in tables. -3. **Chart Cleanup**: Ensure the Chart.js instances are destroyed and re-created cleanly when switching data sources. +3. **Global Filters**: Add checkbox filtering to slice data by Model, Scenario, and Oversight level dynamically on the frontend. +4. **Chart Cleanup**: Ensure the Chart.js instances are destroyed and re-created cleanly when switching data sources or filters. --- @@ -154,8 +130,8 @@ Backend dependencies added via `uv`: [dependencies] fastapi = ">=0.115.0" uvicorn = {extras = ["standard"], version = ">=0.32.0"} -sse-starlette = ">=2.0.0" ``` +*(sse-starlette was removed as live log streaming is no longer required)* Frontend uses `Chart.js` via CDN. No other external dependencies are required. @@ -175,11 +151,13 @@ uv run api/server.py ## Features Implemented -- [x] Run Trigger and Cancellation -- [x] Live log tailing via SSE +- [x] Pure Data Visualization Dashboard - [x] High-contrast Dark Mode ("Control Room" aesthetic) - [x] Dynamic File Selection for CSV/Judge Reports -- [x] Interactive Chart.js Visualizations (Pie & Bar charts based on columns) +- [x] Multi-select Checkbox Global Filters (Model, Scenario, Oversight) +- [x] Interactive Chart.js Visualizations (Deception Rates, Categories) +- [x] Average & Total Token Usage Visualizations (Prompt vs Completion) +- [x] Click-to-Zoom interactive charts (Modal overlay) - [x] Responsive data tables with word-wrapping - [x] Modular ES6 Javascript (No build step) -- cgit v1.2.3