tracebility-ai · gaurav0107 · Jun 6, 2026 · Jun 6, 2026
diff --git a/services/api/tracebility_api/routers/luna_judges.py b/services/api/tracebility_api/routers/luna_judges.py
@@ -54,7 +54,9 @@
 
 router = APIRouter(prefix="/v1/luna-judges", tags=["luna-judges"])
 
-_VALID_PROVIDERS = {"anthropic", "openai", "stub"}
+_VALID_PROVIDERS = {
+    "anthropic", "openai", "gemini", "mistral", "deepseek", "groq", "stub",
+}
 _VALID_FORMATS = {"score-rationale", "json-object"}
 _SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
 _VAR_RE = re.compile(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}")

diff --git a/web/src/components/JudgesClient.tsx b/web/src/components/JudgesClient.tsx
@@ -4,6 +4,8 @@ import { Plus, Trash2 } from "lucide-react";
 import { useRouter } from "next/navigation";
 import { useState, useTransition } from "react";
 
+import { ProviderModelPicker } from "@/components/ModelPicker";
+
 /**
  * Client controls for /judges:
  *   - NewJudgeButton: modal that POSTs /api/luna-judges
@@ -22,20 +24,21 @@ export interface LunaJudgeRow {
   description: string | null;
   rubric_prompt: string;
   output_format: "score-rationale" | "json-object";
-  provider: "anthropic" | "openai" | "stub";
+  provider:
+    | "anthropic"
+    | "openai"
+    | "gemini"
+    | "mistral"
+    | "deepseek"
+    | "groq"
+    | "stub";
   model: string;
   temperature: number | null;
   max_tokens: number;
   created_at: string;
   updated_at: string;
 }
 
-const PROVIDER_OPTIONS = [
-  { value: "anthropic", label: "anthropic" },
-  { value: "openai", label: "openai" },
-  { value: "stub", label: "stub (test only)" },
-] as const;
-
 const FORMAT_OPTIONS = [
   {
     value: "score-rationale",
@@ -68,7 +71,9 @@ export function NewJudgeButton({ projectId }: { projectId: string }) {
   const [name, setName] = useState("");
   const [description, setDescription] = useState("");
   const [rubric, setRubric] = useState(DEFAULT_RUBRIC);
-  const [provider, setProvider] = useState<LunaJudgeRow["provider"]>("anthropic");
+  const [provider, setProvider] = useState<
+    Exclude<LunaJudgeRow["provider"], "stub">
+  >("anthropic");
   const [model, setModel] = useState("claude-sonnet-4-6");
   const [temperature, setTemperature] = useState("0.0");
   const [maxTokens, setMaxTokens] = useState("512");
@@ -231,26 +236,22 @@ export function NewJudgeButton({ projectId }: { projectId: string }) {
             style={{ fontFamily: "var(--font-mono)", fontSize: 12 }}
           />
         </Field>
-        <div style={{ display: "grid", gridTemplateColumns: "1fr 2fr", gap: 12 }}>
-          <Field label="Provider">
-            <select
-              value={provider}
-              onChange={(e) => setProvider(e.target.value as LunaJudgeRow["provider"])}
-            >
-              {PROVIDER_OPTIONS.map((p) => (
-                <option key={p.value} value={p.value}>
-                  {p.label}
-                </option>
-              ))}
-            </select>
-          </Field>
-          <Field label="Model" hint="e.g. claude-sonnet-4-6, gpt-4o-mini, stub-echo">
-            <input
-              value={model}
-              onChange={(e) => setModel(e.target.value)}
-            />
-          </Field>
-        </div>
+        <ProviderModelPicker
+          provider={provider}
+          model={model}
+          onChange={({ provider: p, model: m }) => {
+            setProvider(p);
+            setModel(m);
+          }}
+        />
+        <p
+          className="mono"
+          style={{ fontSize: 11, color: "var(--text-3)", margin: 0 }}
+        >
+          tip: pick &lsquo;Custom&hellip;&rsquo; and type{" "}
+          <code>stub/echo</code> for the deterministic test path that
+          bypasses LiteLLM.
+        </p>
         <div style={{ display: "grid", gridTemplateColumns: "1fr 1fr 2fr", gap: 12 }}>
           <Field label="Temperature">
             <input

diff --git a/web/src/components/ModelPicker.tsx b/web/src/components/ModelPicker.tsx
@@ -0,0 +1,183 @@
+"use client";
+
+import { useState } from "react";
+
+import {
+  ALL_MODELS,
+  MODEL_CATALOG,
+  PROVIDERS,
+  type Provider,
+  findModelOption,
+  providerFromValue,
+} from "@/lib/models";
+
+/**
+ * Provider-grouped model picker.
+ *
+ * Renders an <optgroup> per provider with the curated catalog plus a
+ * "custom..." option that flips to a free-text input. Uncontrolled
+ * value below the catalog is preserved as-is (so a typed
+ * `gemini/gemini-experimental-1206` round-trips even if it's not in
+ * the catalog yet). The escape hatch is the point: we curate for
+ * discoverability, not enforcement.
+ *
+ * The actual gateway dispatches `<provider>/<model_id>`. Legacy bare
+ * names (`gpt-4o`, `claude-sonnet-4`) still work for back-compat with
+ * existing playground sessions; the api's _resolve_provider routes
+ * them. New picks via this component are always fully-qualified.
+ */
+export function ModelPicker({
+  value,
+  onChange,
+  label,
+  ariaLabel,
+}: {
+  value: string;
+  onChange: (next: string) => void;
+  /** Optional label for the field wrapper. If omitted, render just the inputs. */
+  label?: string;
+  ariaLabel?: string;
+}) {
+  // If the current value isn't in the catalog, expose a free-text mode
+  // so the user can edit it directly without losing it.
+  const inCatalog = ALL_MODELS.some((m) => m.value === value);
+  const initialCustom = value !== "" && !inCatalog;
+  const [customMode, setCustomMode] = useState(initialCustom);
+
+  const select = (
+    <select
+      aria-label={ariaLabel ?? "model"}
+      value={customMode ? "__custom__" : value}
+      onChange={(e) => {
+        const next = e.target.value;
+        if (next === "__custom__") {
+          setCustomMode(true);
+          // Don't blow away the existing value when entering custom
+          // mode; the user might have typed something already.
+          return;
+        }
+        setCustomMode(false);
+        onChange(next);
+      }}
+      style={{ width: "100%" }}
+    >
+      {PROVIDERS.map((p) => (
+        <optgroup key={p.value} label={p.label}>
+          {MODEL_CATALOG[p.value].map((m) => (
+            <option key={m.value} value={m.value}>
+              {m.label}
+              {m.hint ? ` — ${m.hint}` : ""}
+            </option>
+          ))}
+        </optgroup>
+      ))}
+      <optgroup label="Other">
+        <option value="__custom__">Custom… (type any model)</option>
+      </optgroup>
+    </select>
+  );
+
+  const customInput = customMode ? (
+    <input
+      aria-label="custom model identifier"
+      value={value}
+      onChange={(e) => onChange(e.target.value)}
+      placeholder="provider/model-id"
+      className="mono"
+      style={{ width: "100%", marginTop: 6, fontSize: 13 }}
+    />
+  ) : null;
+
+  const inferredProvider = providerFromValue(value);
+  const opt = findModelOption(value);
+  const meta = customMode ? (
+    <span
+      className="mono"
+      style={{ fontSize: 11, color: "var(--text-3)", marginTop: 4 }}
+    >
+      {inferredProvider
+        ? `routed to ${inferredProvider}`
+        : "use 'provider/model-id' so the gateway can route correctly"}
+    </span>
+  ) : opt?.hint ? (
+    <span
+      style={{ fontSize: 11, color: "var(--text-3)", marginTop: 4 }}
+    >
+      {opt.provider} · {opt.hint}
+    </span>
+  ) : null;
+
+  if (label) {
+    return (
+      <label style={{ display: "grid", gap: 4 }}>
+        <span
+          style={{
+            fontSize: 11,
+            color: "var(--text-3)",
+            textTransform: "uppercase",
+            letterSpacing: 0.4,
+          }}
+        >
+          {label}
+        </span>
+        {select}
+        {customInput}
+        {meta}
+      </label>
+    );
+  }
+
+  return (
+    <div style={{ display: "grid", gap: 4 }}>
+      {select}
+      {customInput}
+      {meta}
+    </div>
+  );
+}
+
+/**
+ * Compact provider+model split. Used by judge config / variant config
+ * shapes that store {provider, model} as separate columns. Dispatches
+ * the bare model id (no slash prefix) on the wire. Internally this is
+ * the same picker UI; we just split the chosen value.
+ */
+export function ProviderModelPicker({
+  provider,
+  model,
+  onChange,
+}: {
+  provider: Provider | "" | null;
+  model: string;
+  onChange: (next: { provider: Provider; model: string }) => void;
+}) {
+  // Compose the catalog-shaped value; if the model has no provider
+  // prefix, pair it with the supplied provider for select rendering.
+  const composed =
+    model && provider
+      ? model.startsWith(`${provider}/`)
+        ? model
+        : `${provider}/${model}`
+      : "";
+  return (
+    <ModelPicker
+      label="Model"
+      value={composed}
+      onChange={(next) => {
+        const slash = next.indexOf("/");
+        if (slash > 0) {
+          const p = next.slice(0, slash) as Provider;
+          const m = next.slice(slash + 1);
+          if (PROVIDERS.some((pp) => pp.value === p)) {
+            onChange({ provider: p, model: m });
+            return;
+          }
+        }
+        // Custom value with no recognizable prefix: keep the existing
+        // provider, dispatch the raw model. The api will return
+        // bad_model if it's truly unknown, surfaced on the row.
+        if (provider) onChange({ provider, model: next });
+      }}
+    />
+  );
+}
diff --git a/web/src/components/PlaygroundClient.tsx b/web/src/components/PlaygroundClient.tsx
@@ -5,6 +5,8 @@ import Link from "next/link";
 import { useRouter } from "next/navigation";
 import { useMemo, useState, useTransition } from "react";
 
+import { ModelPicker } from "@/components/ModelPicker";
+
 /**
  * Interactive Playground canvas.
  *
@@ -53,15 +55,6 @@ export interface PlaygroundSessionOut {
 
 const VAR_RE = /\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}/g;
 
-const PRESET_MODELS: { value: string; label: string }[] = [
-  { value: "claude-opus-4-7", label: "claude-opus-4-7" },
-  { value: "claude-sonnet-4-6", label: "claude-sonnet-4-6" },
-  { value: "claude-haiku-4-5-20251001", label: "claude-haiku-4-5" },
-  { value: "gpt-4o", label: "gpt-4o" },
-  { value: "gpt-4o-mini", label: "gpt-4o-mini" },
-  { value: "stub-echo", label: "stub-echo (no api key)" },
-];
-
 function extractVariables(template: string): string[] {
   const out = new Set<string>();
   let match: RegExpExecArray | null;
@@ -89,8 +82,8 @@ export function PlaygroundComposer({
   const [variables, setVariables] = useState<Record<string, string>>({
     text: "tracebility is a self-hosted LLM observability platform.",
   });
-  const [model, setModel] = useState<string>("stub-echo");
-  const [modelB, setModelB] = useState<string>("claude-haiku-4-5-20251001");
+  const [model, setModel] = useState<string>("anthropic/claude-sonnet-4-6");
+  const [modelB, setModelB] = useState<string>("openai/gpt-4o-mini");
   const [temperature, setTemperature] = useState<string>("0.7");
   const [maxTokens, setMaxTokens] = useState<string>("1024");
   const [result, setResult] = useState<PlaygroundSessionOut | null>(null);
@@ -499,25 +492,13 @@ function ModelCard({
           gap: 12,
         }}
       >
-        <Field label={mode === "compare" ? "Model A" : "Model"}>
-          <select value={model} onChange={(e) => setModel(e.target.value)}>
-            {PRESET_MODELS.map((m) => (
-              <option key={m.value} value={m.value}>
-                {m.label}
-              </option>
-            ))}
-          </select>
-        </Field>
+        <ModelPicker
+          label={mode === "compare" ? "Model A" : "Model"}
+          value={model}
+          onChange={setModel}
+        />
         {mode === "compare" ? (
-          <Field label="Model B">
-            <select value={modelB} onChange={(e) => setModelB(e.target.value)}>
-              {PRESET_MODELS.map((m) => (
-                <option key={m.value} value={m.value}>
-                  {m.label}
-                </option>
-              ))}
-            </select>
-          </Field>
+          <ModelPicker label="Model B" value={modelB} onChange={setModelB} />
         ) : null}
         <Field label="Temperature" hint="0.0..2.0 (blank = provider default)">
           <input