From aee8a0714a8d7385434f5d5dec5f3735ab08eae6 Mon Sep 17 00:00:00 2001 From: su-fen <715041@qq.com> Date: Fri, 24 Jul 2026 16:21:36 +0800 Subject: [PATCH] feat(models): add mainland-China vendor sections to the model catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the models.dev snapshot from the four native provider catalogs (105 models) to 13 sections (233 models), adding deepseek, zhipuai, moonshotai, minimax, stepfun, xiaomi, longcat, alibaba and tencent. These vendors have no app provider type of their own; their sections are consumed exclusively through findCatalogModelAcrossProviders, so models served via Anthropic/OpenAI-compatible relays under any provider type now resolve real context/output limits. - generate-model-catalog.mjs: replace the flat PROVIDERS list with SECTIONS. Each section unions one or more upstream keys (first source wins; CN endpoints preferred where both exist, e.g. alibaba-cn's qwen-max limits over the stale global entry). - Upstream contamination handled at generation time: Bailian (alibaba-cn) and tencent-coding-plan host third-party models (glm/kimi/MiniMax/deepseek) with platform-clamped limits. Ids are deduplicated case-insensitively across sections in declaration order — pure vendor-official catalogs first, contaminated platforms last — and aggregator-prefixed ids (siliconflow/…, kimi/…) are filtered out. Bailian-only historical deployments (deepseek-v3, deepseek-r1, distills) are kept intentionally: relays commonly serve those ids. - modelCatalog.ts (mirrored): the lookup index adds lowercase aliases so mixed-case catalog ids (MiniMax-M2.5, LongCat-2.0) match lowercase configs; generation-time lowercase dedup guarantees the aliases are unambiguous. - Tests: section keys/order and per-section quality gates locked to the generator's SECTIONS; global uniqueness upgraded to lowercase-uniqueness; cross-provider and settings-defaults cases for the new vendors. models.dev does not list Baidu ERNIE or ByteDance Doubao; those can be picked up by a snapshot refresh once upstream adds them. Co-Authored-By: Claude Fable 5 --- .../web/src/lib/models/catalog.generated.ts | 150 ++++++++++++++++- .../web/src/lib/models/modelCatalog.ts | 19 ++- .../src/lib/models/catalog.generated.ts | 150 ++++++++++++++++- .../agent-gui/src/lib/models/modelCatalog.ts | 19 ++- .../test/models/model-catalog.test.mjs | 44 ++++- .../test/settings/normalization.test.mjs | 10 ++ scripts/generate-model-catalog.mjs | 152 ++++++++++++------ 7 files changed, 473 insertions(+), 71 deletions(-) diff --git a/crates/agent-gateway/web/src/lib/models/catalog.generated.ts b/crates/agent-gateway/web/src/lib/models/catalog.generated.ts index 6ee9ae42f..3a0b1a88e 100644 --- a/crates/agent-gateway/web/src/lib/models/catalog.generated.ts +++ b/crates/agent-gateway/web/src/lib/models/catalog.generated.ts @@ -1,5 +1,5 @@ // Generated by scripts/generate-model-catalog.mjs — DO NOT EDIT. -// Source: https://models.dev/api.json (providers: anthropic, google, openai, xai) +// Source: https://models.dev/api.json (sections: anthropic, google, openai, xai, deepseek, zhipuai, moonshotai, minimax, stepfun, xiaomi, longcat, alibaba, tencent) // Refresh: make update-model-catalog export type CatalogModelEntry = { @@ -8,7 +8,7 @@ export type CatalogModelEntry = { maxOutputToken: number; }; -export type CatalogProviderId = "anthropic" | "google" | "openai" | "xai"; +export type CatalogProviderId = "anthropic" | "google" | "openai" | "xai" | "deepseek" | "zhipuai" | "moonshotai" | "minimax" | "stepfun" | "xiaomi" | "longcat" | "alibaba" | "tencent"; export const MODEL_CATALOG_SNAPSHOT_DATE = "2026-07-24"; @@ -126,4 +126,150 @@ export const MODEL_CATALOG: Record { let index = catalogIndexByProvider.get(catalogProvider); if (!index) { - index = new Map(MODEL_CATALOG[catalogProvider].map((entry) => [entry.id, entry])); + index = new Map(); + for (const entry of MODEL_CATALOG[catalogProvider]) { + index.set(entry.id, entry); + // 目录含混合大小写 id（MiniMax-M2/LongCat-2.0 等）：补小写别名，让候选链 + // 的 lower 候选可命中；生成期按小写去重保证别名不会跨条目歧义。 + const lower = entry.id.toLowerCase(); + if (!index.has(lower)) index.set(lower, entry); + } catalogIndexByProvider.set(catalogProvider, index); } return index; @@ -95,10 +102,12 @@ export function findCatalogModel( return undefined; } -// 中转聚合常把 A 家模型挂在 B 家供应商类型下（如 Anthropic 兼容中转供 grok）， -// 供应商作用域查不到时按 id 跨供应商回查，避免真实限额被本供应商兜底值顶掉。 -// 目录 id 无跨供应商重名（目录不变量测试锁死）；候选链放外层——更精确的 id -// 形态优先于供应商声明序。 +// 中转聚合常把 A 家模型挂在 B 家供应商类型下（grok/deepseek/glm/qwen 等挂在 +// Anthropic/OpenAI 兼容中转），供应商作用域查不到时按 id 跨供应商回查，避免 +// 真实限额被本供应商兜底值顶掉。国内厂商分区（deepseek/zhipuai/alibaba 等） +// 没有对应的应用供应商类型，只经这里消费。目录 id 全局小写唯一（生成期跨 +// 分区去重+目录不变量测试锁死）；候选链放外层——更精确的 id 形态优先于 +// 供应商声明序。 const CATALOG_PROVIDER_IDS = Object.keys(MODEL_CATALOG) as CatalogProviderId[]; export function findCatalogModelAcrossProviders( diff --git a/crates/agent-gui/src/lib/models/catalog.generated.ts b/crates/agent-gui/src/lib/models/catalog.generated.ts index 6ee9ae42f..3a0b1a88e 100644 --- a/crates/agent-gui/src/lib/models/catalog.generated.ts +++ b/crates/agent-gui/src/lib/models/catalog.generated.ts @@ -1,5 +1,5 @@ // Generated by scripts/generate-model-catalog.mjs — DO NOT EDIT. -// Source: https://models.dev/api.json (providers: anthropic, google, openai, xai) +// Source: https://models.dev/api.json (sections: anthropic, google, openai, xai, deepseek, zhipuai, moonshotai, minimax, stepfun, xiaomi, longcat, alibaba, tencent) // Refresh: make update-model-catalog export type CatalogModelEntry = { @@ -8,7 +8,7 @@ export type CatalogModelEntry = { maxOutputToken: number; }; -export type CatalogProviderId = "anthropic" | "google" | "openai" | "xai"; +export type CatalogProviderId = "anthropic" | "google" | "openai" | "xai" | "deepseek" | "zhipuai" | "moonshotai" | "minimax" | "stepfun" | "xiaomi" | "longcat" | "alibaba" | "tencent"; export const MODEL_CATALOG_SNAPSHOT_DATE = "2026-07-24"; @@ -126,4 +126,150 @@ export const MODEL_CATALOG: Record { let index = catalogIndexByProvider.get(catalogProvider); if (!index) { - index = new Map(MODEL_CATALOG[catalogProvider].map((entry) => [entry.id, entry])); + index = new Map(); + for (const entry of MODEL_CATALOG[catalogProvider]) { + index.set(entry.id, entry); + // 目录含混合大小写 id（MiniMax-M2/LongCat-2.0 等）：补小写别名，让候选链 + // 的 lower 候选可命中；生成期按小写去重保证别名不会跨条目歧义。 + const lower = entry.id.toLowerCase(); + if (!index.has(lower)) index.set(lower, entry); + } catalogIndexByProvider.set(catalogProvider, index); } return index; @@ -95,10 +102,12 @@ export function findCatalogModel( return undefined; } -// 中转聚合常把 A 家模型挂在 B 家供应商类型下（如 Anthropic 兼容中转供 grok）， -// 供应商作用域查不到时按 id 跨供应商回查，避免真实限额被本供应商兜底值顶掉。 -// 目录 id 无跨供应商重名（目录不变量测试锁死）；候选链放外层——更精确的 id -// 形态优先于供应商声明序。 +// 中转聚合常把 A 家模型挂在 B 家供应商类型下（grok/deepseek/glm/qwen 等挂在 +// Anthropic/OpenAI 兼容中转），供应商作用域查不到时按 id 跨供应商回查，避免 +// 真实限额被本供应商兜底值顶掉。国内厂商分区（deepseek/zhipuai/alibaba 等） +// 没有对应的应用供应商类型，只经这里消费。目录 id 全局小写唯一（生成期跨 +// 分区去重+目录不变量测试锁死）；候选链放外层——更精确的 id 形态优先于 +// 供应商声明序。 const CATALOG_PROVIDER_IDS = Object.keys(MODEL_CATALOG) as CatalogProviderId[]; export function findCatalogModelAcrossProviders( diff --git a/crates/agent-gui/test/models/model-catalog.test.mjs b/crates/agent-gui/test/models/model-catalog.test.mjs index e737be981..c3565e218 100644 --- a/crates/agent-gui/test/models/model-catalog.test.mjs +++ b/crates/agent-gui/test/models/model-catalog.test.mjs @@ -5,18 +5,38 @@ import { createTsModuleLoader } from "../helpers/load-ts-module.mjs"; const loader = createTsModuleLoader(); const catalog = loader.loadModule("src/lib/models/modelCatalog.ts"); -const PROVIDERS = ["anthropic", "google", "openai", "xai"]; -// 与 scripts/generate-model-catalog.mjs 的质量门同值：上游被截断时刷新会硬错， -// 这里锁住已入库快照的完整性。 -const MIN_MODELS_PER_PROVIDER = { anthropic: 8, google: 15, openai: 20, xai: 3 }; +// 与 scripts/generate-model-catalog.mjs 的 SECTIONS 同值（键、序、质量门）： +// 上游被截断时刷新会硬错，这里锁住已入库快照的完整性。前四家是应用供应商 +// 类型的原生目录；其余为国内厂商分区，只经跨供应商回查消费。 +const MIN_MODELS_PER_PROVIDER = { + anthropic: 8, + google: 15, + openai: 20, + xai: 3, + deepseek: 3, + zhipuai: 10, + moonshotai: 8, + minimax: 5, + stepfun: 4, + xiaomi: 4, + longcat: 1, + alibaba: 40, + tencent: 4, +}; +const PROVIDERS = Object.keys(MIN_MODELS_PER_PROVIDER); test("generated catalog upholds the data invariants", () => { - // 跨供应商回查（findCatalogModelAcrossProviders）依赖 id 全目录唯一， - // 否则同名模型在不同供应商下会产生歧义命中。 + assert.deepEqual( + Object.keys(catalog.MODEL_CATALOG), + PROVIDERS, + "catalog sections must match the generator's SECTIONS (keys and order)", + ); + // 跨供应商回查（findCatalogModelAcrossProviders）与索引的小写别名依赖 + // id 全目录按小写唯一，否则同名模型在不同分区下会产生歧义命中。 const allIds = PROVIDERS.flatMap((providerId) => - catalog.MODEL_CATALOG[providerId].map((entry) => entry.id), + catalog.MODEL_CATALOG[providerId].map((entry) => entry.id.toLowerCase()), ); - assert.equal(new Set(allIds).size, allIds.length, "ids must be unique across providers"); + assert.equal(new Set(allIds).size, allIds.length, "ids must be lowercase-unique across sections"); for (const providerId of PROVIDERS) { const entries = catalog.MODEL_CATALOG[providerId]; assert.ok( @@ -102,6 +122,14 @@ test("cross-provider lookup resolves models configured under a foreign provider" maxOutputToken: 32_000, }); assert.equal(catalog.resolveModelLimitsAcrossProviders("model-not-in-catalog"), undefined); + // 国内厂商分区（无对应应用供应商类型）经跨供应商回查可命中。 + assert.equal(catalog.findCatalogModelAcrossProviders("deepseek-chat")?.id, "deepseek-chat"); + assert.equal(catalog.findCatalogModelAcrossProviders("glm-4.6")?.id, "glm-4.6"); + assert.equal(catalog.findCatalogModelAcrossProviders("qwen-max")?.id, "qwen-max"); + assert.equal(catalog.findCatalogModelAcrossProviders("kimi-k2.5")?.id, "kimi-k2.5"); + // 混合大小写目录 id（MiniMax/LongCat）：小写配置经索引别名命中，返回原始 id。 + assert.equal(catalog.findCatalogModelAcrossProviders("minimax-m2.5")?.id, "MiniMax-M2.5"); + assert.equal(catalog.findCatalogModelAcrossProviders("longcat-2.0")?.id, "LongCat-2.0"); }); test("repairStaleCrossProviderLimits replaces only stale provider-fallback pairs", () => { diff --git a/crates/agent-gui/test/settings/normalization.test.mjs b/crates/agent-gui/test/settings/normalization.test.mjs index 212301e4a..8e7d3a75d 100644 --- a/crates/agent-gui/test/settings/normalization.test.mjs +++ b/crates/agent-gui/test/settings/normalization.test.mjs @@ -2419,6 +2419,16 @@ test("cross-provider models resolve real catalog limits instead of provider fall settings.getProviderModelDefaults("claude_code", "GROK-4.5@prod").contextWindow, 500_000, ); + // 国内厂商模型（deepseek/glm/qwen/kimi/MiniMax 等分区）没有自己的应用供应商 + // 类型，配在任一类型下都经跨供应商回查取真实限额。 + const deepseekUnderClaude = settings.getProviderModelDefaults("claude_code", "deepseek-chat"); + assert.equal(deepseekUnderClaude.contextWindow, 1_000_000); + assert.equal(deepseekUnderClaude.maxOutputToken, 384_000); + const glmUnderCodex = settings.getProviderModelDefaults("codex", "glm-4.7"); + assert.equal(glmUnderCodex.contextWindow, 204_800); + assert.equal(glmUnderCodex.maxOutputToken, 131_072); + // 混合大小写目录 id（MiniMax-M2.1）：小写配置经索引小写别名命中。 + assert.equal(settings.getProviderModelDefaults("codex", "minimax-m2.1").contextWindow, 204_800); // 全目录未收录的模型仍吃本供应商兜底值。 assert.equal( settings.getProviderModelDefaults("claude_code", "some-custom-model").contextWindow, diff --git a/scripts/generate-model-catalog.mjs b/scripts/generate-model-catalog.mjs index bbc3d6f8d..9362ab626 100644 --- a/scripts/generate-model-catalog.mjs +++ b/scripts/generate-model-catalog.mjs @@ -24,15 +24,50 @@ const OUTPUT_PATHS = [ ]; const DEFAULT_SOURCE = "https://models.dev/api.json"; -// Fixed provider order: these are the native catalogs behind the app's four -// provider types (claude_code→anthropic, gemini→google, codex→openai, xai). -const PROVIDERS = ["anthropic", "google", "openai", "xai"]; -// Guard against a truncated upstream response replacing a healthy snapshot. -const MIN_MODELS_PER_PROVIDER = { anthropic: 8, google: 15, openai: 20, xai: 3 }; -// Models that must exist; their absence signals an upstream schema change. + +// Catalog sections. Each section unions one or more upstream models.dev +// provider keys (first source wins on a same-id conflict — used to prefer the +// China endpoint's limits for vendors that publish both). +// +// The first four sections are the native catalogs behind the app's provider +// types (claude_code→anthropic, gemini→google, codex→openai, xai); scoped +// lookup (findCatalogModel) only ever reads these. The remaining sections are +// mainland-China vendors with no app provider type of their own: they are +// consumed exclusively through findCatalogModelAcrossProviders, so models +// served through claude_code/codex-compatible relays resolve real limits. +// +// Section order is also the adjudication order for duplicate ids across +// sections: ids are deduplicated case-insensitively, first section wins. +// Pure vendor-official catalogs therefore come first; "alibaba" (Bailian) and +// "tencent" (coding plan) host third-party models (glm/kimi/MiniMax/deepseek +// deployments with platform-clamped limits), so they come last and their +// copies of another vendor's models are dropped in favor of the official ones. +const SECTIONS = [ + { key: "anthropic", sources: ["anthropic"], min: 8 }, + { key: "google", sources: ["google"], min: 15 }, + { key: "openai", sources: ["openai"], min: 20 }, + { key: "xai", sources: ["xai"], min: 3 }, + { key: "deepseek", sources: ["deepseek"], min: 3 }, + // zai (Z.AI, international brand) is a superset of zhipuai with identical + // ids and limits for the overlap; keep the domestic brand as the key. + { key: "zhipuai", sources: ["zai", "zhipuai"], min: 10 }, + { key: "moonshotai", sources: ["moonshotai-cn", "moonshotai"], min: 8 }, + { key: "minimax", sources: ["minimax-cn", "minimax"], min: 5 }, + { key: "stepfun", sources: ["stepfun"], min: 4 }, + { key: "xiaomi", sources: ["xiaomi"], min: 4 }, + { key: "longcat", sources: ["longcat"], min: 1 }, + { key: "alibaba", sources: ["alibaba-cn", "alibaba"], min: 40 }, + { key: "tencent", sources: ["tencent-coding-plan"], min: 4 }, +]; + +// Models that must exist; their absence signals an upstream schema change +// (or, for unioned sections, a source-key rename). const SENTINELS = [ ["anthropic", "claude-sonnet-4-6"], ["openai", "gpt-5"], + ["deepseek", "deepseek-chat"], + ["zhipuai", "glm-4.6"], + ["alibaba", "qwen-max"], ]; // Single semantic rule shared with lib/models/modelCatalog.ts (bound together @@ -91,35 +126,54 @@ async function loadUpstream(source) { return undefined; } -function extractModels(providerId, providerData) { - const rawModels = providerData?.models; - if (!rawModels || typeof rawModels !== "object") { - fail(`provider ${providerId}: missing models map`); - } +// claimedLower: lowercased id -> owning section key. Lowercase-unique ids +// across the whole catalog are what make cross-provider lookup unambiguous +// and let the runtime index add case-insensitive aliases; the invariant is +// re-asserted by test/models/model-catalog.test.mjs. +function extractSection(section, upstream, claimedLower) { const entries = []; - const seen = new Set(); - for (const [id, model] of Object.entries(rawModels)) { - const contextWindow = model?.limit?.context; - const rawOutput = model?.limit?.output; - if (!model?.modalities?.output?.includes?.("text")) { - console.error(`skip ${providerId}/${id} (non-text output)`); - continue; - } - if (!Number.isInteger(contextWindow) || contextWindow <= 0) { - console.error(`skip ${providerId}/${id} (invalid limit.context)`); - continue; + for (const source of section.sources) { + const providerData = upstream?.[source]; + if (!providerData) fail(`section ${section.key}: source ${source} missing from upstream data`); + const rawModels = providerData.models; + if (!rawModels || typeof rawModels !== "object") { + fail(`section ${section.key}: source ${source} missing models map`); } - if (!Number.isInteger(rawOutput) || rawOutput <= 0) { - console.error(`skip ${providerId}/${id} (invalid limit.output)`); - continue; + for (const [id, model] of Object.entries(rawModels)) { + // Aggregator-namespaced deployments (e.g. Bailian's "siliconflow/…", + // "kimi/…") are not vendor model ids; relays never serve them verbatim. + if (id.includes("/")) { + console.error(`skip ${source}/${id} (aggregator-prefixed id)`); + continue; + } + const contextWindow = model?.limit?.context; + const rawOutput = model?.limit?.output; + if (!model?.modalities?.output?.includes?.("text")) { + console.error(`skip ${source}/${id} (non-text output)`); + continue; + } + if (!Number.isInteger(contextWindow) || contextWindow <= 0) { + console.error(`skip ${source}/${id} (invalid limit.context)`); + continue; + } + if (!Number.isInteger(rawOutput) || rawOutput <= 0) { + console.error(`skip ${source}/${id} (invalid limit.output)`); + continue; + } + const lower = id.toLowerCase(); + const claimedBy = claimedLower.get(lower); + if (claimedBy === section.key) continue; // CN/global union overlap: first source wins. + if (claimedBy) { + console.error(`skip ${source}/${id} (id claimed by section ${claimedBy})`); + continue; + } + claimedLower.set(lower, section.key); + entries.push({ + id, + contextWindow, + maxOutputToken: normalizeMaxOutputToken(contextWindow, rawOutput), + }); } - if (seen.has(id)) fail(`provider ${providerId}: duplicate model id ${id}`); - seen.add(id); - entries.push({ - id, - contextWindow, - maxOutputToken: normalizeMaxOutputToken(contextWindow, rawOutput), - }); } entries.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); return entries; @@ -130,9 +184,10 @@ function renderEntry(entry) { } function renderCatalog(catalog, snapshotDate) { + const keys = SECTIONS.map((section) => section.key); const lines = [ "// Generated by scripts/generate-model-catalog.mjs — DO NOT EDIT.", - "// Source: https://models.dev/api.json (providers: anthropic, google, openai, xai)", + `// Source: https://models.dev/api.json (sections: ${keys.join(", ")})`, "// Refresh: make update-model-catalog", "", "export type CatalogModelEntry = {", @@ -141,15 +196,15 @@ function renderCatalog(catalog, snapshotDate) { " maxOutputToken: number;", "};", "", - `export type CatalogProviderId = ${PROVIDERS.map((p) => JSON.stringify(p)).join(" | ")};`, + `export type CatalogProviderId = ${keys.map((key) => JSON.stringify(key)).join(" | ")};`, "", `export const MODEL_CATALOG_SNAPSHOT_DATE = "${snapshotDate}";`, "", "export const MODEL_CATALOG: Record = {", ]; - for (const providerId of PROVIDERS) { - lines.push(` ${providerId}: [`); - for (const entry of catalog[providerId]) lines.push(renderEntry(entry)); + for (const key of keys) { + lines.push(` ${key}: [`); + for (const entry of catalog[key]) lines.push(renderEntry(entry)); lines.push(" ],"); } lines.push("};", ""); @@ -172,21 +227,20 @@ const args = parseArgs(process.argv); const upstream = await loadUpstream(args.source); const catalog = {}; -for (const providerId of PROVIDERS) { - const providerData = upstream?.[providerId]; - if (!providerData) fail(`provider ${providerId}: missing from upstream data`); - const entries = extractModels(providerId, providerData); - if (entries.length < MIN_MODELS_PER_PROVIDER[providerId]) { +const claimedLower = new Map(); +for (const section of SECTIONS) { + const entries = extractSection(section, upstream, claimedLower); + if (entries.length < section.min) { fail( - `provider ${providerId}: only ${entries.length} models after filtering ` + - `(expected >= ${MIN_MODELS_PER_PROVIDER[providerId]}); upstream data looks truncated`, + `section ${section.key}: only ${entries.length} models after filtering ` + + `(expected >= ${section.min}); upstream data looks truncated`, ); } - catalog[providerId] = entries; + catalog[section.key] = entries; } -for (const [providerId, modelId] of SENTINELS) { - if (!catalog[providerId].some((entry) => entry.id === modelId)) { - fail(`sentinel model ${providerId}/${modelId} missing; upstream schema may have changed`); +for (const [sectionKey, modelId] of SENTINELS) { + if (!catalog[sectionKey].some((entry) => entry.id === modelId)) { + fail(`sentinel model ${sectionKey}/${modelId} missing; upstream schema may have changed`); } } @@ -218,5 +272,5 @@ if (unchanged) { for (const path of OUTPUT_PATHS) writeFileSync(path, nextContent); const [guiBytes, webBytes] = OUTPUT_PATHS.map((path) => readFileSync(path)); if (!guiBytes.equals(webBytes)) fail("post-write self-check failed: outputs differ"); -const total = PROVIDERS.reduce((sum, providerId) => sum + catalog[providerId].length, 0); +const total = SECTIONS.reduce((sum, section) => sum + catalog[section.key].length, 0); console.log(`catalog updated (${total} models, snapshot ${today})`);