diff --git a/apps/web/src/app/admin/auto-routing/AutoRoutingAdminContent.tsx b/apps/web/src/app/admin/auto-routing/AutoRoutingAdminContent.tsx index f6e262d43d..e677cdd9ef 100644 --- a/apps/web/src/app/admin/auto-routing/AutoRoutingAdminContent.tsx +++ b/apps/web/src/app/admin/auto-routing/AutoRoutingAdminContent.tsx @@ -452,6 +452,9 @@ export function AutoRoutingAdminContent() { void classifierModelQuery.refetch(); void analyticsQuery.refetch(); void openRouterModelsQuery.refetch(); + // Invalidate the shared selector catalog the benchmark pickers read + // from (query key prefix ['openrouter-models']). + void queryClient.invalidateQueries({ queryKey: ['openrouter-models'] }); }} disabled={isRefreshing} className="w-fit" diff --git a/apps/web/src/app/admin/auto-routing/BenchmarksSection.test.ts b/apps/web/src/app/admin/auto-routing/BenchmarksSection.test.ts index ad3a402e67..874deb212e 100644 --- a/apps/web/src/app/admin/auto-routing/BenchmarksSection.test.ts +++ b/apps/web/src/app/admin/auto-routing/BenchmarksSection.test.ts @@ -10,7 +10,9 @@ import { formatAccuracy, formatUsd, formStateToConfig, + pinnedModelFor, RoutingTableView, + variantOptionsForModel, } from './BenchmarksSection'; describe('formatAccuracy', () => { @@ -76,6 +78,36 @@ describe('costPerAccuracy', () => { }); }); +describe('pinnedModelFor', () => { + it('pins a saved id as a selectable option', () => { + expect(pinnedModelFor('anthropic/claude-sonnet-4.5')).toEqual({ + id: 'anthropic/claude-sonnet-4.5', + name: 'anthropic/claude-sonnet-4.5', + }); + }); +}); + +describe('variantOptionsForModel', () => { + it('offers the selected model catalog variant keys', () => { + const option = { id: 'openai/gpt-5', name: 'GPT-5', variants: ['none', 'low', 'high'] }; + expect(variantOptionsForModel(option, null)).toEqual(['none', 'low', 'high']); + }); + + it('appends a saved variant key omitted from the catalog', () => { + const option = { id: 'openai/gpt-5', name: 'GPT-5', variants: ['low', 'high'] }; + expect(variantOptionsForModel(option, 'max')).toEqual(['low', 'high', 'max']); + }); + + it('does not duplicate a saved variant key that is still in the catalog', () => { + const option = { id: 'openai/gpt-5', name: 'GPT-5', variants: ['low', 'high'] }; + expect(variantOptionsForModel(option, 'high')).toEqual(['low', 'high']); + }); + + it('hides when neither catalog nor saved variant key exists', () => { + expect(variantOptionsForModel({ id: 'model', name: 'Model' }, null)).toEqual([]); + }); +}); + describe('RoutingTableView', () => { it('renders candidates in the published serving rank order', () => { const html = renderToStaticMarkup( @@ -115,7 +147,7 @@ describe('RoutingTableView', () => { expect(html.indexOf('threshold-meeting')).toBeLessThan(html.indexOf('below-threshold-cheaper')); }); - it('renders reasoning effort next to the model name', () => { + it('renders canonical variant before effort/default', () => { const html = renderToStaticMarkup( React.createElement(RoutingTableView, { data: { @@ -134,6 +166,40 @@ describe('RoutingTableView', () => { accuracy: 0.8, avgCostUsd: 0.006, meetsThreshold: true, + variant: 'xhigh', + }, + ], + }, + }, + }, + }) + ); + + expect(html.indexOf('Model')).toBeLessThan(html.indexOf('Variant')); + expect(html.indexOf('Variant')).toBeLessThan(html.indexOf('Accuracy')); + expect(html.indexOf('openai/gpt-5')).toBeLessThan(html.indexOf('xhigh')); + expect(html.indexOf('xhigh')).toBeLessThan(html.indexOf('80.0%')); + }); + + it('renders legacy reasoning effort as the variant label when variant is absent', () => { + const html = renderToStaticMarkup( + React.createElement(RoutingTableView, { + data: { + publishedAt: '2026-06-17T00:00:00.000Z', + table: { + version: 'run-1', + generatedAt: '2026-06-17T00:00:00.000Z', + minAccuracy: 0.7, + switchCostFactor: 3, + bestAccuracySwitchThreshold: 0.05, + source: 'benchmark', + routes: { + 'implementation/code_generation': [ + { + model: 'anthropic/claude-sonnet-4.5', + accuracy: 0.8, + avgCostUsd: 0.006, + meetsThreshold: true, reasoningEffort: 'high', }, ], @@ -143,11 +209,39 @@ describe('RoutingTableView', () => { }) ); - expect(html.indexOf('Model')).toBeLessThan(html.indexOf('Reasoning effort')); - expect(html.indexOf('Reasoning effort')).toBeLessThan(html.indexOf('Accuracy')); - expect(html.indexOf('openai/gpt-5')).toBeLessThan(html.indexOf('high')); + expect(html.indexOf('anthropic/claude-sonnet-4.5')).toBeLessThan(html.indexOf('high')); expect(html.indexOf('high')).toBeLessThan(html.indexOf('80.0%')); }); + + it('renders default when neither variant nor effort exists', () => { + const html = renderToStaticMarkup( + React.createElement(RoutingTableView, { + data: { + publishedAt: '2026-06-17T00:00:00.000Z', + table: { + version: 'run-1', + generatedAt: '2026-06-17T00:00:00.000Z', + minAccuracy: 0.7, + switchCostFactor: 3, + bestAccuracySwitchThreshold: 0.05, + source: 'benchmark', + routes: { + 'implementation/code_generation': [ + { + model: 'openai/gpt-4o-mini', + accuracy: 0.8, + avgCostUsd: 0.006, + meetsThreshold: true, + }, + ], + }, + }, + }, + }) + ); + + expect(html.indexOf('openai/gpt-4o-mini')).toBeLessThan(html.indexOf('default')); + }); }); describe('configToFormState', () => { @@ -158,7 +252,7 @@ describe('configToFormState', () => { expect(state.classifierMaxP95LatencyMs).toBe('1000'); expect(state.autoDeciderMinCostUsd).toBe(15); expect(state.autoDeciderMaxCostUsd).toBe(25); - expect(state.classifierModels).toBe(''); + expect(state.classifierModels).toEqual([]); expect(state.deciderModels).toEqual([]); expect(state.autoDeciderModels).toEqual([]); expect(state.excludedAutoDeciderModels).toBe(''); @@ -201,7 +295,7 @@ describe('formStateToConfig round-trip', () => { expect(state.autoDeciderMinCostUsd).toBe(12); expect(state.autoDeciderMaxCostUsd).toBe(24); expect(state.benchmarkOrgId).toBe('org-123'); - expect(state.deciderModels).toEqual([{ id: 'manual-model', reasoningEffort: 'low' }]); + expect(state.deciderModels).toEqual([{ id: 'manual-model', variant: 'low' }]); expect(state.autoDeciderModels).toEqual(baseConfig.autoDeciderModels); expect(state.excludedAutoDeciderModels).toBe('excluded-auto-model'); @@ -212,14 +306,67 @@ describe('formStateToConfig round-trip', () => { expect(result.autoDeciderMinCostUsd).toBe(12); expect(result.autoDeciderMaxCostUsd).toBe(24); expect(result.benchmarkOrgId).toBe('org-123'); - expect(result.manualDeciderModels).toEqual([{ id: 'manual-model', reasoningEffort: 'low' }]); + expect(result.manualDeciderModels).toEqual([ + { id: 'manual-model', variant: 'low', reasoningEffort: null }, + ]); expect(result.excludedAutoDeciderModels).toEqual(['excluded-auto-model']); expect(result.deciderModels).toEqual([ - { id: 'manual-model', reasoningEffort: 'low' }, + { id: 'manual-model', variant: 'low', reasoningEffort: null }, + // Auto rows stay effort-only in the effective list. { id: 'auto-model', reasoningEffort: null }, ]); }); + it('round-trips classifierModels as a string array and drops blank rows', () => { + const state = configToFormState(baseConfig); + expect(state.classifierModels).toEqual(['model-a', 'model-b']); + const result = formStateToConfig( + { ...state, classifierModels: ['model-a', ' ', 'model-c'] }, + baseConfig + ); + expect(result.classifierModels).toEqual(['model-a', 'model-c']); + }); + + it('round-trips canonical variant rows and loads legacy effort rows as variant', () => { + const variantConfig: BenchmarkConfig = { + ...baseConfig, + manualDeciderModels: [ + { id: 'manual-v2', variant: 'high', reasoningEffort: null }, + { id: 'legacy', reasoningEffort: 'low' }, + ], + }; + const state = configToFormState(variantConfig); + expect(state.deciderModels).toEqual([ + { id: 'manual-v2', variant: 'high' }, + { id: 'legacy', variant: 'low' }, + ]); + + const result = formStateToConfig(state, variantConfig); + expect(result.manualDeciderModels).toEqual([ + { id: 'manual-v2', variant: 'high', reasoningEffort: null }, + { id: 'legacy', variant: 'low', reasoningEffort: null }, + ]); + // Manual rows save variant-only; the legacy effort field stays null. + expect(result.manualDeciderModels?.every(model => model.reasoningEffort === null)).toBe(true); + }); + + it('drops blank decider rows on save', () => { + const state = configToFormState(baseConfig); + const result = formStateToConfig( + { + ...state, + deciderModels: [ + { id: ' ', variant: null }, + { id: 'manual-model', variant: 'low' }, + ], + }, + baseConfig + ); + expect(result.manualDeciderModels).toEqual([ + { id: 'manual-model', variant: 'low', reasoningEffort: null }, + ]); + }); + it('converts empty-string classifierMaxP95LatencyMs form value to null in config', () => { const state = configToFormState(baseConfig); const stateWithEmpty = { ...state, classifierMaxP95LatencyMs: '' }; @@ -239,24 +386,35 @@ describe('formStateToConfig round-trip', () => { }); describe('effectiveDeciderModels', () => { - it('combines manual models with non-excluded auto models and lets manual override an auto duplicate', () => { + it('keeps manual rows variant-only and auto rows effort-only, drops excluded auto, and lets manual override an auto duplicate', () => { expect( effectiveDeciderModels({ manualDeciderModels: [ - { id: 'manual/model', reasoningEffort: null }, - { id: 'auto/duplicate', reasoningEffort: 'high' }, + { id: 'manual/model', variant: null }, + { id: 'auto/duplicate', variant: 'high' }, ], autoDeciderModels: [ { id: 'auto/duplicate', reasoningEffort: null, avgAttemptCostUsd: 20 }, { id: 'auto/included', reasoningEffort: 'low', avgAttemptCostUsd: 22 }, { id: 'auto/excluded', reasoningEffort: null, avgAttemptCostUsd: 23 }, + { + id: 'auto/variant-carrying', + variant: 'xhigh', + reasoningEffort: null, + avgAttemptCostUsd: 24, + }, ], excludedAutoDeciderModels: ['auto/excluded'], }) - ).toEqual([ - { id: 'manual/model', reasoningEffort: null }, - { id: 'auto/duplicate', reasoningEffort: 'high' }, + ).toStrictEqual([ + // Manual rows are canonical variant-only with the legacy effort null. + { id: 'manual/model', variant: null, reasoningEffort: null }, + // A manual row with the same id overrides the synced auto row. + { id: 'auto/duplicate', variant: 'high', reasoningEffort: null }, + // Auto rows stay effort-only and never emit the variant key, even when + // the synced row carries a variant. { id: 'auto/included', reasoningEffort: 'low' }, + { id: 'auto/variant-carrying', reasoningEffort: null }, ]); }); }); diff --git a/apps/web/src/app/admin/auto-routing/BenchmarksSection.tsx b/apps/web/src/app/admin/auto-routing/BenchmarksSection.tsx index 621a52e67b..5c58923347 100644 --- a/apps/web/src/app/admin/auto-routing/BenchmarksSection.tsx +++ b/apps/web/src/app/admin/auto-routing/BenchmarksSection.tsx @@ -9,32 +9,29 @@ import { DEFAULT_BENCHMARK_ORG_ID, DEFAULT_BENCHMARK_USER_ID, StartBenchmarkRunResponseSchema, + type AutoBenchmarkDeciderModel, type BenchmarkConfig, + type BenchmarkDeciderModel, type BenchmarkKind, + type BenchmarkModelSummary, type BenchmarkRoutingTableResponse, type BenchmarkRun, - type BenchmarkModelSummary, type RankedCandidate, - type ReasoningEffort, - type AutoBenchmarkDeciderModel, } from '@kilocode/auto-routing-contracts'; -import React, { useCallback, useEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { toast } from 'sonner'; import { ChevronDown, ChevronRight, Play, Plus, Save, Trash2 } from 'lucide-react'; +import { useModelSelectorList } from '@/app/api/openrouter/hooks'; +import { toEligibleModelOptions } from '@/components/auto-routing/AutoRoutingModeCard'; +import { ModelCombobox, type ModelOption } from '@/components/shared/ModelCombobox'; +import { VariantCombobox } from '@/components/shared/VariantCombobox'; import { Badge } from '@/components/ui/badge'; import { Button } from '@/components/ui/button'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; import { Checkbox } from '@/components/ui/checkbox'; import { Input } from '@/components/ui/input'; import { Label } from '@/components/ui/label'; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/components/ui/select'; import { Skeleton } from '@/components/ui/skeleton'; import { Table, @@ -44,7 +41,6 @@ import { TableHeader, TableRow, } from '@/components/ui/table'; -import { Textarea } from '@/components/ui/textarea'; import { parseAdminResponse } from './admin-fetch'; // --------------------------------------------------------------------------- @@ -75,6 +71,31 @@ export function formatCostPerAccuracy(candidate: Pick ({ id: m.id, - reasoningEffort: m.reasoningEffort ?? null, + // Legacy reasoningEffort-only rows load as the canonical form variant so + // a save emits variant-only manual rows. + variant: m.variant ?? m.reasoningEffort ?? null, })), autoDeciderModels: config.autoDeciderModels ?? [], excludedAutoDeciderModels: (config.excludedAutoDeciderModels ?? []).join('\n'), @@ -202,12 +225,15 @@ export function effectiveDeciderModels({ manualDeciderModels: DeciderModelRow[]; autoDeciderModels: AutoDeciderModelRow[]; excludedAutoDeciderModels: string[]; -}): DeciderModelRow[] { +}): BenchmarkDeciderModel[] { const manual = manualDeciderModels .filter(row => row.id.trim().length > 0) .map(row => ({ id: row.id.trim(), - reasoningEffort: row.reasoningEffort ?? null, + variant: row.variant ?? null, + // The contract type requires the legacy field; it stays null on + // variant-only rows (both non-null is malformed). + reasoningEffort: null, })); const manualIds = new Set(manual.map(model => model.id)); const excludedAuto = new Set(excludedAutoDeciderModels); @@ -218,6 +244,9 @@ export function effectiveDeciderModels({ .filter(model => !manualIds.has(model.id)) .map(model => ({ id: model.id, + // Auto rows stay effort-only: the benchmark worker writes and reads the + // legacy reasoningEffort for synced rows, so never emit the canonical + // variant key here. reasoningEffort: model.reasoningEffort ?? null, })), ]; @@ -227,11 +256,17 @@ export function formStateToConfig( state: ReturnType, base: BenchmarkConfig | null ): BenchmarkConfig { - const classifierModels = parseModelLines(state.classifierModels); + const classifierModels = state.classifierModels.map(id => id.trim()).filter(id => id.length > 0); const excludedAutoDeciderModels = parseModelLines(state.excludedAutoDeciderModels); const manualDeciderModels = state.deciderModels .filter(row => row.id.trim().length > 0) - .map(row => ({ id: row.id.trim(), reasoningEffort: row.reasoningEffort ?? null })); + .map(row => ({ + id: row.id.trim(), + variant: row.variant ?? null, + // Variant-only rows: the legacy effort field is always null so the two + // never collide (both non-null is malformed per the contract). + reasoningEffort: null, + })); const deciderModels = effectiveDeciderModels({ manualDeciderModels, autoDeciderModels: state.autoDeciderModels, @@ -270,9 +305,15 @@ export function formStateToConfig( function BenchmarkConfigEditor({ config, onSaved, + modelOptions, + modelsLoading, + modelsError, }: { config: BenchmarkConfig | null; onSaved: (next: { config: BenchmarkConfig | null }) => void; + modelOptions: ModelOption[]; + modelsLoading: boolean; + modelsError?: string; }) { const [form, setForm] = useState(() => configToFormState(config)); // Tracks unsaved local edits. A background config refetch (the runs list @@ -319,10 +360,37 @@ function BenchmarkConfigEditor({ }, }); + const handleAddClassifierRow = useCallback(() => { + updateForm(prev => ({ + ...prev, + classifierModels: [...prev.classifierModels, ''], + })); + }, [updateForm]); + + const handleRemoveClassifierRow = useCallback( + (index: number) => { + updateForm(prev => ({ + ...prev, + classifierModels: prev.classifierModels.filter((_, i) => i !== index), + })); + }, + [updateForm] + ); + + const handleClassifierModelChange = useCallback( + (index: number, value: string) => { + updateForm(prev => ({ + ...prev, + classifierModels: prev.classifierModels.map((id, i) => (i === index ? value : id)), + })); + }, + [updateForm] + ); + const handleAddDeciderRow = useCallback(() => { updateForm(prev => ({ ...prev, - deciderModels: [...prev.deciderModels, { id: '', reasoningEffort: null }], + deciderModels: [...prev.deciderModels, { id: '', variant: null }], })); }, [updateForm]); @@ -378,66 +446,34 @@ function BenchmarkConfigEditor({ {/* Classifier models */}
- -