From 334101938e242efa4561325fc79076b9fb72aad4 Mon Sep 17 00:00:00 2001
From: gaurav0107 <gauravdubey0107@gmail.com>
Date: Sun, 7 Jun 2026 02:26:21 +0530
Subject: [PATCH] feat(web): provider-grouped ModelPicker + widen Luna provider
 check
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Static MODEL_CATALOG per provider in web/src/lib/models.ts (4-6 models
each: anthropic/openai/gemini/mistral/deepseek/groq). New
<ModelPicker> renders a grouped <select> with a 'Custom...' escape
hatch for models not yet in the catalog.

Wired into:
- Playground (single + compare modes) via ModelPicker
- /judges create form via ProviderModelPicker (provider+model split)

api: widen luna_judges _VALID_PROVIDERS to include the four new ones.

Discoverability fix for the Lite­LLM PR: configured a Gemini key but
'how do I pick a Gemini model' had no answer. The picker shows them
grouped by provider; users still type custom IDs through the escape
hatch.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .../tracebility_api/routers/luna_judges.py    |   4 +-
 web/src/components/JudgesClient.tsx           |  57 +++---
 web/src/components/ModelPicker.tsx            | 183 ++++++++++++++++++
 web/src/components/PlaygroundClient.tsx       |  39 +---
 web/src/lib/models.ts                         | 103 ++++++++++
 5 files changed, 328 insertions(+), 58 deletions(-)
 create mode 100644 web/src/components/ModelPicker.tsx
 create mode 100644 web/src/lib/models.ts

diff --git a/services/api/tracebility_api/routers/luna_judges.py b/services/api/tracebility_api/routers/luna_judges.py
index 066ed96..b4013e3 100644
--- a/services/api/tracebility_api/routers/luna_judges.py
+++ b/services/api/tracebility_api/routers/luna_judges.py
@@ -54,7 +54,9 @@
 
 router = APIRouter(prefix="/v1/luna-judges", tags=["luna-judges"])
 
-_VALID_PROVIDERS = {"anthropic", "openai", "stub"}
+_VALID_PROVIDERS = {
+    "anthropic", "openai", "gemini", "mistral", "deepseek", "groq", "stub",
+}
 _VALID_FORMATS = {"score-rationale", "json-object"}
 _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
 _VAR_RE = re.compile(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}")
diff --git a/web/src/components/JudgesClient.tsx b/web/src/components/JudgesClient.tsx
index c64afb3..c90c48a 100644
--- a/web/src/components/JudgesClient.tsx
+++ b/web/src/components/JudgesClient.tsx
@@ -4,6 +4,8 @@ import { Plus, Trash2 } from "lucide-react";
 import { useRouter } from "next/navigation";
 import { useState, useTransition } from "react";
 
+import { ProviderModelPicker } from "@/components/ModelPicker";
+
 /**
  * Client controls for /judges:
  *   - NewJudgeButton: modal that POSTs /api/luna-judges
@@ -22,7 +24,14 @@ export interface LunaJudgeRow {
   description: string | null;
   rubric_prompt: string;
   output_format: "score-rationale" | "json-object";
-  provider: "anthropic" | "openai" | "stub";
+  provider:
+    | "anthropic"
+    | "openai"
+    | "gemini"
+    | "mistral"
+    | "deepseek"
+    | "groq"
+    | "stub";
   model: string;
   temperature: number | null;
   max_tokens: number;
@@ -30,12 +39,6 @@ export interface LunaJudgeRow {
   updated_at: string;
 }
 
-const PROVIDER_OPTIONS = [
-  { value: "anthropic", label: "anthropic" },
-  { value: "openai", label: "openai" },
-  { value: "stub", label: "stub (test only)" },
-] as const;
-
 const FORMAT_OPTIONS = [
   {
     value: "score-rationale",
@@ -68,7 +71,9 @@ export function NewJudgeButton({ projectId }: { projectId: string }) {
   const [name, setName] = useState("");
   const [description, setDescription] = useState("");
   const [rubric, setRubric] = useState(DEFAULT_RUBRIC);
-  const [provider, setProvider] = useState<LunaJudgeRow["provider"]>("anthropic");
+  const [provider, setProvider] = useState<
+    Exclude<LunaJudgeRow["provider"], "stub">
+  >("anthropic");
   const [model, setModel] = useState("claude-sonnet-4-6");
   const [temperature, setTemperature] = useState("0.0");
   const [maxTokens, setMaxTokens] = useState("512");
@@ -231,26 +236,22 @@ export function NewJudgeButton({ projectId }: { projectId: string }) {
             style={{ fontFamily: "var(--font-mono)", fontSize: 12 }}
           />
         </Field>
-        <div style={{ display: "grid", gridTemplateColumns: "1fr 2fr", gap: 12 }}>
-          <Field label="Provider">
-            <select
-              value={provider}
-              onChange={(e) => setProvider(e.target.value as LunaJudgeRow["provider"])}
-            >
-              {PROVIDER_OPTIONS.map((p) => (
-                <option key={p.value} value={p.value}>
-                  {p.label}
-                </option>
-              ))}
-            </select>
-          </Field>
-          <Field label="Model" hint="e.g. claude-sonnet-4-6, gpt-4o-mini, stub-echo">
-            <input
-              value={model}
-              onChange={(e) => setModel(e.target.value)}
-            />
-          </Field>
-        </div>
+        <ProviderModelPicker
+          provider={provider}
+          model={model}
+          onChange={({ provider: p, model: m }) => {
+            setProvider(p);
+            setModel(m);
+          }}
+        />
+        <p
+          className="mono"
+          style={{ fontSize: 11, color: "var(--text-3)", margin: 0 }}
+        >
+          tip: pick &lsquo;Custom&hellip;&rsquo; and type{" "}
+          <code>stub/echo</code> for the deterministic test path that
+          bypasses LiteLLM.
+        </p>
         <div style={{ display: "grid", gridTemplateColumns: "1fr 1fr 2fr", gap: 12 }}>
           <Field label="Temperature">
             <input
diff --git a/web/src/components/ModelPicker.tsx b/web/src/components/ModelPicker.tsx
new file mode 100644
index 0000000..84d38d5
--- /dev/null
+++ b/web/src/components/ModelPicker.tsx
@@ -0,0 +1,183 @@
+"use client";
+
+import { useState } from "react";
+
+import {
+  ALL_MODELS,
+  MODEL_CATALOG,
+  PROVIDERS,
+  type Provider,
+  findModelOption,
+  providerFromValue,
+} from "@/lib/models";
+
+/**
+ * Provider-grouped model picker.
+ *
+ * Renders an <optgroup> per provider with the curated catalog plus a
+ * "custom..." option that flips to a free-text input. Uncontrolled
+ * value below the catalog is preserved as-is (so a typed
+ * `gemini/gemini-experimental-1206` round-trips even if it's not in
+ * the catalog yet). The escape hatch is the point: we curate for
+ * discoverability, not enforcement.
+ *
+ * The actual gateway dispatches `<provider>/<model_id>`. Legacy bare
+ * names (`gpt-4o`, `claude-sonnet-4`) still work for back-compat with
+ * existing playground sessions; the api's _resolve_provider routes
+ * them. New picks via this component are always fully-qualified.
+ */
+export function ModelPicker({
+  value,
+  onChange,
+  label,
+  ariaLabel,
+}: {
+  value: string;
+  onChange: (next: string) => void;
+  /** Optional label for the field wrapper. If omitted, render just the inputs. */
+  label?: string;
+  ariaLabel?: string;
+}) {
+  // If the current value isn't in the catalog, expose a free-text mode
+  // so the user can edit it directly without losing it.
+  const inCatalog = ALL_MODELS.some((m) => m.value === value);
+  const initialCustom = value !== "" && !inCatalog;
+  const [customMode, setCustomMode] = useState(initialCustom);
+
+  const select = (
+    <select
+      aria-label={ariaLabel ?? "model"}
+      value={customMode ? "__custom__" : value}
+      onChange={(e) => {
+        const next = e.target.value;
+        if (next === "__custom__") {
+          setCustomMode(true);
+          // Don't blow away the existing value when entering custom
+          // mode; the user might have typed something already.
+          return;
+        }
+        setCustomMode(false);
+        onChange(next);
+      }}
+      style={{ width: "100%" }}
+    >
+      {PROVIDERS.map((p) => (
+        <optgroup key={p.value} label={p.label}>
+          {MODEL_CATALOG[p.value].map((m) => (
+            <option key={m.value} value={m.value}>
+              {m.label}
+              {m.hint ? ` — ${m.hint}` : ""}
+            </option>
+          ))}
+        </optgroup>
+      ))}
+      <optgroup label="Other">
+        <option value="__custom__">Custom… (type any model)</option>
+      </optgroup>
+    </select>
+  );
+
+  const customInput = customMode ? (
+    <input
+      aria-label="custom model identifier"
+      value={value}
+      onChange={(e) => onChange(e.target.value)}
+      placeholder="provider/model-id"
+      className="mono"
+      style={{ width: "100%", marginTop: 6, fontSize: 13 }}
+    />
+  ) : null;
+
+  const inferredProvider = providerFromValue(value);
+  const opt = findModelOption(value);
+  const meta = customMode ? (
+    <span
+      className="mono"
+      style={{ fontSize: 11, color: "var(--text-3)", marginTop: 4 }}
+    >
+      {inferredProvider
+        ? `routed to ${inferredProvider}`
+        : "use 'provider/model-id' so the gateway can route correctly"}
+    </span>
+  ) : opt?.hint ? (
+    <span
+      style={{ fontSize: 11, color: "var(--text-3)", marginTop: 4 }}
+    >
+      {opt.provider} · {opt.hint}
+    </span>
+  ) : null;
+
+  if (label) {
+    return (
+      <label style={{ display: "grid", gap: 4 }}>
+        <span
+          style={{
+            fontSize: 11,
+            color: "var(--text-3)",
+            textTransform: "uppercase",
+            letterSpacing: 0.4,
+          }}
+        >
+          {label}
+        </span>
+        {select}
+        {customInput}
+        {meta}
+      </label>
+    );
+  }
+
+  return (
+    <div style={{ display: "grid", gap: 4 }}>
+      {select}
+      {customInput}
+      {meta}
+    </div>
+  );
+}
+
+/**
+ * Compact provider+model split. Used by judge config / variant config
+ * shapes that store {provider, model} as separate columns. Dispatches
+ * the bare model id (no slash prefix) on the wire. Internally this is
+ * the same picker UI; we just split the chosen value.
+ */
+export function ProviderModelPicker({
+  provider,
+  model,
+  onChange,
+}: {
+  provider: Provider | "" | null;
+  model: string;
+  onChange: (next: { provider: Provider; model: string }) => void;
+}) {
+  // Compose the catalog-shaped value; if the model has no provider
+  // prefix, pair it with the supplied provider for select rendering.
+  const composed =
+    model && provider
+      ? model.startsWith(`${provider}/`)
+        ? model
+        : `${provider}/${model}`
+      : "";
+  return (
+    <ModelPicker
+      label="Model"
+      value={composed}
+      onChange={(next) => {
+        const slash = next.indexOf("/");
+        if (slash > 0) {
+          const p = next.slice(0, slash) as Provider;
+          const m = next.slice(slash + 1);
+          if (PROVIDERS.some((pp) => pp.value === p)) {
+            onChange({ provider: p, model: m });
+            return;
+          }
+        }
+        // Custom value with no recognizable prefix: keep the existing
+        // provider, dispatch the raw model. The api will return
+        // bad_model if it's truly unknown, surfaced on the row.
+        if (provider) onChange({ provider, model: next });
+      }}
+    />
+  );
+}
diff --git a/web/src/components/PlaygroundClient.tsx b/web/src/components/PlaygroundClient.tsx
index 24160bf..6fa92f5 100644
--- a/web/src/components/PlaygroundClient.tsx
+++ b/web/src/components/PlaygroundClient.tsx
@@ -5,6 +5,8 @@ import Link from "next/link";
 import { useRouter } from "next/navigation";
 import { useMemo, useState, useTransition } from "react";
 
+import { ModelPicker } from "@/components/ModelPicker";
+
 /**
  * Interactive Playground canvas.
  *
@@ -53,15 +55,6 @@ export interface PlaygroundSessionOut {
 
 const VAR_RE = /\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}/g;
 
-const PRESET_MODELS: { value: string; label: string }[] = [
-  { value: "claude-opus-4-7", label: "claude-opus-4-7" },
-  { value: "claude-sonnet-4-6", label: "claude-sonnet-4-6" },
-  { value: "claude-haiku-4-5-20251001", label: "claude-haiku-4-5" },
-  { value: "gpt-4o", label: "gpt-4o" },
-  { value: "gpt-4o-mini", label: "gpt-4o-mini" },
-  { value: "stub-echo", label: "stub-echo (no api key)" },
-];
-
 function extractVariables(template: string): string[] {
   const out = new Set<string>();
   let match: RegExpExecArray | null;
@@ -89,8 +82,8 @@ export function PlaygroundComposer({
   const [variables, setVariables] = useState<Record<string, string>>({
     text: "tracebility is a self-hosted LLM observability platform.",
   });
-  const [model, setModel] = useState<string>("stub-echo");
-  const [modelB, setModelB] = useState<string>("claude-haiku-4-5-20251001");
+  const [model, setModel] = useState<string>("anthropic/claude-sonnet-4-6");
+  const [modelB, setModelB] = useState<string>("openai/gpt-4o-mini");
   const [temperature, setTemperature] = useState<string>("0.7");
   const [maxTokens, setMaxTokens] = useState<string>("1024");
   const [result, setResult] = useState<PlaygroundSessionOut | null>(null);
@@ -499,25 +492,13 @@ function ModelCard({
           gap: 12,
         }}
       >
-        <Field label={mode === "compare" ? "Model A" : "Model"}>
-          <select value={model} onChange={(e) => setModel(e.target.value)}>
-            {PRESET_MODELS.map((m) => (
-              <option key={m.value} value={m.value}>
-                {m.label}
-              </option>
-            ))}
-          </select>
-        </Field>
+        <ModelPicker
+          label={mode === "compare" ? "Model A" : "Model"}
+          value={model}
+          onChange={setModel}
+        />
         {mode === "compare" ? (
-          <Field label="Model B">
-            <select value={modelB} onChange={(e) => setModelB(e.target.value)}>
-              {PRESET_MODELS.map((m) => (
-                <option key={m.value} value={m.value}>
-                  {m.label}
-                </option>
-              ))}
-            </select>
-          </Field>
+          <ModelPicker label="Model B" value={modelB} onChange={setModelB} />
         ) : null}
         <Field label="Temperature" hint="0.0..2.0 (blank = provider default)">
           <input
diff --git a/web/src/lib/models.ts b/web/src/lib/models.ts
new file mode 100644
index 0000000..83340bc
--- /dev/null
+++ b/web/src/lib/models.ts
@@ -0,0 +1,103 @@
+/**
+ * Static model catalog per provider.
+ *
+ * The gateway accepts model strings of the form `<provider>/<id>`. This
+ * catalog is what the UI shows in pickers — a curated set that's known
+ * to work end-to-end through LiteLLM. New models can be typed in by
+ * hand (the picker has a "custom..." escape hatch); the catalog is
+ * about discoverability, not a hard allow-list.
+ *
+ * The `value` field is what gets sent to the api. For the playground's
+ * legacy bare-name path (e.g. `gpt-4o`, `claude-sonnet-4`) the gateway
+ * accepts both forms and `_resolve_provider` derives the provider from
+ * the prefix.
+ */
+
+export type Provider =
+  | "anthropic"
+  | "openai"
+  | "gemini"
+  | "mistral"
+  | "deepseek"
+  | "groq";
+
+export interface ModelOption {
+  /** What gets dispatched. `<provider>/<model_id>`. */
+  value: string;
+  /** What the user sees. Short. */
+  label: string;
+  /** One-line tier hint. ≤32 chars. */
+  hint?: string;
+}
+
+export const PROVIDERS: { value: Provider; label: string }[] = [
+  { value: "anthropic", label: "Anthropic" },
+  { value: "openai", label: "OpenAI" },
+  { value: "gemini", label: "Gemini" },
+  { value: "mistral", label: "Mistral" },
+  { value: "deepseek", label: "DeepSeek" },
+  { value: "groq", label: "Groq" },
+];
+
+/**
+ * Curated model catalog. Keep this list short — 4-6 per provider, the
+ * ones a researcher actually picks between. New releases get added here
+ * by hand (~quarterly). The "custom..." escape hatch in the picker
+ * covers anything not on the list.
+ */
+export const MODEL_CATALOG: Record<Provider, ModelOption[]> = {
+  anthropic: [
+    { value: "anthropic/claude-opus-4-7", label: "claude-opus-4-7", hint: "frontier reasoning" },
+    { value: "anthropic/claude-sonnet-4-6", label: "claude-sonnet-4-6", hint: "balanced" },
+    { value: "anthropic/claude-haiku-4-5-20251001", label: "claude-haiku-4-5", hint: "fast / cheap" },
+  ],
+  openai: [
+    { value: "openai/gpt-4o", label: "gpt-4o", hint: "flagship multimodal" },
+    { value: "openai/gpt-4o-mini", label: "gpt-4o-mini", hint: "cheap" },
+    { value: "openai/o3", label: "o3", hint: "deep reasoning" },
+    { value: "openai/o4-mini", label: "o4-mini", hint: "reasoning, cheap" },
+  ],
+  gemini: [
+    { value: "gemini/gemini-2.5-pro", label: "gemini-2.5-pro", hint: "long context" },
+    { value: "gemini/gemini-2.5-flash", label: "gemini-2.5-flash", hint: "fast" },
+    { value: "gemini/gemini-1.5-pro", label: "gemini-1.5-pro", hint: "1M-token window" },
+    { value: "gemini/gemini-1.5-flash", label: "gemini-1.5-flash", hint: "cheap" },
+  ],
+  mistral: [
+    { value: "mistral/mistral-large-latest", label: "mistral-large", hint: "flagship" },
+    { value: "mistral/mistral-small-latest", label: "mistral-small", hint: "fast" },
+    { value: "mistral/codestral-latest", label: "codestral", hint: "code" },
+  ],
+  deepseek: [
+    { value: "deepseek/deepseek-chat", label: "deepseek-chat", hint: "general" },
+    { value: "deepseek/deepseek-reasoner", label: "deepseek-reasoner", hint: "reasoning" },
+  ],
+  groq: [
+    { value: "groq/llama-3.3-70b-versatile", label: "llama-3.3-70b", hint: "fast hosted" },
+    { value: "groq/llama-3.1-8b-instant", label: "llama-3.1-8b-instant", hint: "cheap" },
+    { value: "groq/mixtral-8x7b-32768", label: "mixtral-8x7b", hint: "32k context" },
+  ],
+};
+
+/** Flattened view: [{provider, ...ModelOption}, ...] */
+export const ALL_MODELS: (ModelOption & { provider: Provider })[] =
+  PROVIDERS.flatMap((p) =>
+    MODEL_CATALOG[p.value].map((m) => ({ ...m, provider: p.value })),
+  );
+
+/** Return the option matching `value`, or undefined if none. */
+export function findModelOption(
+  value: string,
+): (ModelOption & { provider: Provider }) | undefined {
+  return ALL_MODELS.find((m) => m.value === value);
+}
+
+/** Extract provider from a `<provider>/<id>` string. Returns null
+ * for legacy bare names (`gpt-4o`, `claude-sonnet-4`) since those
+ * are routed by the api's _resolve_provider, not by us. */
+export function providerFromValue(value: string): Provider | null {
+  const slash = value.indexOf("/");
+  if (slash <= 0) return null;
+  const prefix = value.slice(0, slash);
+  return PROVIDERS.some((p) => p.value === prefix) ? (prefix as Provider) : null;
+}
