From 7ff1b9c0fdb31835219ca3afa7b58c96d385fdc8 Mon Sep 17 00:00:00 2001 From: DESU-CLUB Date: Tue, 19 May 2026 15:59:58 -0700 Subject: [PATCH 1/4] Lock pipeline contracts (DatasetSchema, DatasetRow, RunManifest) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds zod-validated contracts for Phase 1↔2 seams per project_spec.md. Cross-field invariants: exactly one column with is_primary_key=true, its name matches DatasetSchema.primary_key, it is non-nullable and enumerable. No implementation yet — contracts only. Co-Authored-By: Claude Opus 4.7 --- backend/package-lock.json | 12 +++- backend/package.json | 3 +- backend/src/pipeline/types.ts | 116 ++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 backend/src/pipeline/types.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index 7931616..bae2a76 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -13,7 +13,8 @@ "convex": "^1.39.1", "dotenv": "^16.4.0", "fastify": "^5.0.0", - "fastify-plugin": "^5.1.0" + "fastify-plugin": "^5.1.0", + "zod": "^4.4.3" }, "devDependencies": { "@types/node": "^22.0.0", @@ -1866,6 +1867,15 @@ "optional": true } } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/backend/package.json b/backend/package.json index eb61b32..963e769 100644 --- a/backend/package.json +++ b/backend/package.json @@ -14,7 +14,8 @@ "convex": "^1.39.1", "dotenv": "^16.4.0", "fastify": "^5.0.0", - "fastify-plugin": "^5.1.0" + "fastify-plugin": "^5.1.0", + "zod": "^4.4.3" }, "devDependencies": { "@types/node": "^22.0.0", diff --git a/backend/src/pipeline/types.ts b/backend/src/pipeline/types.ts new file mode 100644 index 0000000..2d5cc58 --- /dev/null +++ b/backend/src/pipeline/types.ts @@ -0,0 +1,116 @@ +import { z } from "zod"; + +export const columnTypeSchema = z.enum([ + "string", + "url", + "date", + "number", + "boolean", + "enum", +]); +export type ColumnType = z.infer; + +export const retrievalStrategySchema = z.enum([ + "search_fetch", + "browser", + "hybrid", +]); +export type RetrievalStrategy = z.infer; + +const snakeCase = /^[a-z][a-z0-9_]*$/; + +export const columnDefinitionSchema = z.object({ + name: z.string().regex(snakeCase, "must be snake_case"), + display_name: z.string().min(1), + type: columnTypeSchema, + is_primary_key: z.boolean(), + is_enumerable: z.boolean(), + retrieval_hint: z.string(), + nullable: z.boolean(), +}); +export type ColumnDefinition = z.infer; + +export const datasetSchemaSchema = z + .object({ + dataset_name: z.string().regex(snakeCase, "must be snake_case"), + description: z.string().min(1), + columns: z.array(columnDefinitionSchema).min(1), + primary_key: z.string(), + retrieval_strategy: retrievalStrategySchema, + source_hint: z.string().min(1), + }) + .superRefine((data, ctx) => { + const names = data.columns.map((c) => c.name); + const dupes = [...new Set(names.filter((n, i) => names.indexOf(n) !== i))]; + if (dupes.length > 0) { + ctx.addIssue({ + code: "custom", + message: `duplicate column names: ${dupes.join(", ")}`, + path: ["columns"], + }); + } + + const pkCols = data.columns.filter((c) => c.is_primary_key); + if (pkCols.length !== 1) { + ctx.addIssue({ + code: "custom", + message: `exactly one column must have is_primary_key=true (found ${pkCols.length})`, + path: ["columns"], + }); + return; + } + + const pk = pkCols[0]; + if (pk.name !== data.primary_key) { + ctx.addIssue({ + code: "custom", + message: `primary_key '${data.primary_key}' does not match the column flagged is_primary_key ('${pk.name}')`, + path: ["primary_key"], + }); + } + if (pk.nullable) { + ctx.addIssue({ + code: "custom", + message: "primary key column must not be nullable", + path: ["columns"], + }); + } + if (!pk.is_enumerable) { + ctx.addIssue({ + code: "custom", + message: "primary key column must have is_enumerable=true", + path: ["columns"], + }); + } + }); +export type DatasetSchema = z.infer; + +export const datasetRowValueSchema = z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), +]); +export type DatasetRowValue = z.infer; + +export const datasetRowSchema = z.record(z.string(), datasetRowValueSchema); +export type DatasetRow = z.infer; + +export const endpointCallSchema = z.object({ + endpoint: z.enum(["search", "fetch", "browser"]), + count: z.number().int().nonnegative(), +}); +export type EndpointCall = z.infer; + +export const runManifestSchema = z.object({ + run_id: z.string(), + prompt: z.string(), + schema_path: z.string(), + dataset_path: z.string(), + csv_path: z.string(), + row_count: z.number().int().nonnegative(), + columns_filled: z.array(z.string()), + created_at: z.string(), + endpoints_called: z.array(endpointCallSchema), +}); +export type RunManifest = z.infer; From e7dfe5829e697af12580515432f2d8dfd29fa10b Mon Sep 17 00:00:00 2001 From: DESU-CLUB Date: Tue, 19 May 2026 16:15:36 -0700 Subject: [PATCH 2/4] =?UTF-8?q?Add=20Phase=201=20schema-inference=20CLI=20?= =?UTF-8?q?(prompt=20=E2=86=92=20DatasetSchema)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements input → schema → primary-key identification per project_spec.md: - `inferSchema(prompt)` calls Claude (claude-sonnet-4-6) with tool-forced JSON output, validates the result against `datasetSchemaSchema`, and retries once with the formatted zod error appended on failure. - CLI entry `npm run infer-schema -- --prompt "..."` writes `backend/output//schema.json`. - System prompt at `backend/prompts/schema-inference.txt`, loaded via `import.meta.url` (ESM-safe, works in dev and built). - `ANTHROPIC_API_KEY` is optional in `env.ts` (so the Fastify server still boots without it) and asserted at module load in `schema-inference.ts` so the CLI fails fast when missing. Placeholder added to `backend/.env.example`. - Generated `backend/output/` is gitignored. Phase 2 (row enumeration, TinyFish tools, Mastra agent) is deferred to a follow-up PR. Co-Authored-By: Claude Opus 4.7 --- .gitignore | 2 + backend/.env.example | 4 + backend/package-lock.json | 51 +++++++++ backend/package.json | 4 +- backend/prompts/schema-inference.txt | 20 ++++ backend/src/cli.ts | 42 ++++++++ backend/src/env.ts | 2 + backend/src/pipeline/schema-inference.ts | 132 +++++++++++++++++++++++ 8 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 backend/prompts/schema-inference.txt create mode 100644 backend/src/cli.ts create mode 100644 backend/src/pipeline/schema-inference.ts diff --git a/.gitignore b/.gitignore index f21d121..f75ce72 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ yarn-debug.log* *.bak tmp/ temp/ + +backend/output/ diff --git a/backend/.env.example b/backend/.env.example index beece7e..7adc01f 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -9,3 +9,7 @@ CONVEX_SELF_HOSTED_ADMIN_KEY= # Required once any user-facing protected route is added. # Same value as the frontend's CLERK_SECRET_KEY. CLERK_SECRET_KEY= + +# Anthropic API key — required by the schema-inference CLI (npm run infer-schema). +# Generate at https://console.anthropic.com/settings/keys +ANTHROPIC_API_KEY=sk-ant-... diff --git a/backend/package-lock.json b/backend/package-lock.json index bae2a76..7cbf088 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -8,6 +8,7 @@ "name": "bigset-backend", "version": "0.1.0", "dependencies": { + "@anthropic-ai/sdk": "^0.97.1", "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", "convex": "^1.39.1", @@ -22,6 +23,36 @@ "typescript": "^5.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.97.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.97.1.tgz", + "integrity": "sha512-wOf7AUeJPitcVpvKO4UMu63mWH5SaVipkGd7OOQJt/G6VYGlV8D2Gp9dLxOrttDJh/9gqPqdaBwDGcBevumeAg==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@clerk/backend": { "version": "3.4.11", "resolved": "https://registry.npmjs.org/@clerk/backend/-/backend-3.4.11.tgz", @@ -1022,6 +1053,19 @@ "dequal": "^2.0.3" } }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -1317,6 +1361,12 @@ "node": ">=12" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -1873,6 +1923,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/backend/package.json b/backend/package.json index 963e769..cbba04b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -6,9 +6,11 @@ "scripts": { "dev": "tsx watch src/index.ts", "build": "tsc", - "start": "node dist/index.js" + "start": "node dist/index.js", + "infer-schema": "tsx src/cli.ts" }, "dependencies": { + "@anthropic-ai/sdk": "^0.97.1", "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", "convex": "^1.39.1", diff --git a/backend/prompts/schema-inference.txt b/backend/prompts/schema-inference.txt new file mode 100644 index 0000000..ff2fc77 --- /dev/null +++ b/backend/prompts/schema-inference.txt @@ -0,0 +1,20 @@ +You are a data engineering assistant that converts natural-language prompts into structured dataset schemas. Given a user prompt describing a dataset they want to build, you emit a single tool call to `emit_dataset_schema` with a precise schema definition. + +Your job is to: + +1. Identify the universe of entities the user wants to collect. Each entity becomes one row in the dataset. +2. Pick a clear primary key — the column whose values uniquely identify each row. This is usually a name, ID, or canonical URL. Exactly one column must have `is_primary_key: true`, and its `name` must equal `primary_key`. The primary key column must have `nullable: false` and `is_enumerable: true`. +3. Choose useful columns. Each column captures one fact about the entity. Use snake_case names. Mark `is_enumerable: true` only on columns whose values can be used to list all rows (typically just the primary key, and occasionally one or two others when a source page lists them alongside the primary key). +4. Set `retrieval_strategy`: + - `search_fetch` — the data lives on a static page or sitemap that can be fetched as HTML. + - `browser` — the source is a JavaScript-heavy SPA, requires scroll/click to reveal items, or paginates client-side. + - `hybrid` — unclear; the pipeline will try search_fetch first and fall back to browser. +5. Set `source_hint` to a specific URL whenever possible (e.g. `https://www.ycombinator.com/companies?industry=Fintech`). Avoid vague descriptions. +6. Write a `retrieval_hint` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row. + +Rules: + +- Return ONLY the tool call. Do not include any prose response. +- `dataset_name` must be snake_case. +- All column `name` values must be snake_case and unique. +- Prefer concrete column choices over speculative ones — better to omit a column than guess wildly. diff --git a/backend/src/cli.ts b/backend/src/cli.ts new file mode 100644 index 0000000..0d14680 --- /dev/null +++ b/backend/src/cli.ts @@ -0,0 +1,42 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { randomBytes } from "node:crypto"; +import { join } from "node:path"; + +import { inferSchema } from "./pipeline/schema-inference.js"; + +function parsePrompt(argv: string[]): string { + const idx = argv.findIndex((a) => a === "--prompt"); + if (idx === -1 || idx === argv.length - 1) { + throw new Error('Usage: npm run infer-schema -- --prompt ""'); + } + const value = argv[idx + 1]; + if (!value.trim()) throw new Error("--prompt requires a non-empty value"); + return value; +} + +function generateRunId(): string { + const ts = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14); + const rand = randomBytes(3).toString("hex"); + return `${ts}-${rand}`; +} + +async function main() { + const prompt = parsePrompt(process.argv.slice(2)); + const runId = generateRunId(); + const outDir = join("output", runId); + + console.log(`Inferring schema for: "${prompt}"`); + const schema = await inferSchema(prompt); + + mkdirSync(outDir, { recursive: true }); + const schemaPath = join(outDir, "schema.json"); + writeFileSync(schemaPath, JSON.stringify(schema, null, 2) + "\n"); + + console.log(`Run ID: ${runId}`); + console.log(`Schema: backend/${schemaPath}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/backend/src/env.ts b/backend/src/env.ts index 62e3ce9..606ab33 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -22,4 +22,6 @@ export const env = { // today because no protected routes exist yet; required as soon as one is // added. CLERK_SECRET_KEY: process.env.CLERK_SECRET_KEY, + + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, }; diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts new file mode 100644 index 0000000..6399cff --- /dev/null +++ b/backend/src/pipeline/schema-inference.ts @@ -0,0 +1,132 @@ +import { readFileSync } from "node:fs"; +import Anthropic from "@anthropic-ai/sdk"; +import { z } from "zod"; + +import { env } from "../env.js"; +import { datasetSchemaSchema, type DatasetSchema } from "./types.js"; + +const MODEL = "claude-sonnet-4-6"; +const TOOL_NAME = "emit_dataset_schema"; + +const SYSTEM_PROMPT = readFileSync( + new URL("../../prompts/schema-inference.txt", import.meta.url), + "utf8", +); + +if (!env.ANTHROPIC_API_KEY) { + throw new Error("Missing required environment variable: ANTHROPIC_API_KEY"); +} + +const client = new Anthropic({ apiKey: env.ANTHROPIC_API_KEY }); + +const datasetSchemaInputSchema = { + type: "object", + required: [ + "dataset_name", + "description", + "columns", + "primary_key", + "retrieval_strategy", + "source_hint", + ], + properties: { + dataset_name: { + type: "string", + description: "snake_case identifier for the dataset", + }, + description: { + type: "string", + description: "Human-readable summary of what this dataset captures", + }, + primary_key: { + type: "string", + description: "Name of the column that uniquely identifies each row", + }, + retrieval_strategy: { + type: "string", + enum: ["search_fetch", "browser", "hybrid"], + description: + "search_fetch for static pages, browser for SPAs/paginated UIs, hybrid if unclear", + }, + source_hint: { + type: "string", + description: + "Specific URL where the data can be found, or a precise description if no single URL exists", + }, + columns: { + type: "array", + minItems: 1, + items: { + type: "object", + required: [ + "name", + "display_name", + "type", + "is_primary_key", + "is_enumerable", + "retrieval_hint", + "nullable", + ], + properties: { + name: { type: "string", description: "snake_case column name" }, + display_name: { type: "string" }, + type: { + type: "string", + enum: ["string", "url", "date", "number", "boolean", "enum"], + }, + is_primary_key: { type: "boolean" }, + is_enumerable: { + type: "boolean", + description: + "true only on the primary key and any other columns whose values can be used to enumerate all rows", + }, + retrieval_hint: { + type: "string", + description: "Where/how to find this column's value in Phase 3", + }, + nullable: { type: "boolean" }, + }, + }, + }, + }, +}; + +export async function inferSchema(prompt: string): Promise { + const first = await callOnce(prompt); + const firstParsed = datasetSchemaSchema.safeParse(first); + if (firstParsed.success) return firstParsed.data; + + const errorText = z.prettifyError(firstParsed.error); + const retryMessage = `${prompt}\n\nYour previous output failed validation:\n${errorText}\n\nReturn a corrected DatasetSchema.`; + const second = await callOnce(retryMessage); + return datasetSchemaSchema.parse(second); +} + +async function callOnce(userMessage: string): Promise { + const response = await client.messages.create({ + model: MODEL, + max_tokens: 4096, + system: [ + { + type: "text", + text: SYSTEM_PROMPT, + cache_control: { type: "ephemeral" }, + }, + ], + tools: [ + { + name: TOOL_NAME, + description: + "Emit the inferred DatasetSchema describing the dataset to be built.", + input_schema: datasetSchemaInputSchema as Anthropic.Tool["input_schema"], + }, + ], + tool_choice: { type: "tool", name: TOOL_NAME }, + messages: [{ role: "user", content: userMessage }], + }); + + for (const block of response.content) { + if (block.type === "tool_use") return block.input; + } + throw new Error("Model did not emit a tool_use block"); +} From f544da71cfaeca3d201c791d8e7dfa42bae03c0a Mon Sep 17 00:00:00 2001 From: Simantak Dabhade Date: Wed, 20 May 2026 12:47:59 -0700 Subject: [PATCH 3/4] Swap Anthropic SDK for Vercel AI SDK + OpenRouter Replaces @anthropic-ai/sdk with ai (Vercel AI SDK 6) and @openrouter/ai-sdk-provider so the schema-inference CLI is not locked to a single model provider. Uses generateText + Output.object() which passes the Zod schema directly, eliminating the duplicate JSON Schema definition and manual tool_use extraction. API key check moved from module-level to getModel() so importing this module won't crash the backend server when OPENROUTER_API_KEY is unset. Co-Authored-By: Claude Opus 4.6 --- backend/.env.example | 6 +- backend/package-lock.json | 149 ++++++++++++++++------- backend/package.json | 3 +- backend/prompts/schema-inference.txt | 3 +- backend/src/env.ts | 2 +- backend/src/pipeline/schema-inference.ts | 146 +++++----------------- 6 files changed, 146 insertions(+), 163 deletions(-) diff --git a/backend/.env.example b/backend/.env.example index 7adc01f..adfa070 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -10,6 +10,6 @@ CONVEX_SELF_HOSTED_ADMIN_KEY= # Same value as the frontend's CLERK_SECRET_KEY. CLERK_SECRET_KEY= -# Anthropic API key — required by the schema-inference CLI (npm run infer-schema). -# Generate at https://console.anthropic.com/settings/keys -ANTHROPIC_API_KEY=sk-ant-... +# OpenRouter API key — required by the schema-inference CLI (npm run infer-schema). +# Generate at https://openrouter.ai/settings/keys +OPENROUTER_API_KEY=sk-or-... diff --git a/backend/package-lock.json b/backend/package-lock.json index 7cbf088..730bd2f 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -8,9 +8,10 @@ "name": "bigset-backend", "version": "0.1.0", "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", + "@openrouter/ai-sdk-provider": "^2.9.0", + "ai": "^6.0.0", "convex": "^1.39.1", "dotenv": "^16.4.0", "fastify": "^5.0.0", @@ -23,34 +24,50 @@ "typescript": "^5.0.0" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.97.1", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.97.1.tgz", - "integrity": "sha512-wOf7AUeJPitcVpvKO4UMu63mWH5SaVipkGd7OOQJt/G6VYGlV8D2Gp9dLxOrttDJh/9gqPqdaBwDGcBevumeAg==", - "license": "MIT", + "node_modules/@ai-sdk/gateway": { + "version": "3.0.116", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.116.tgz", + "integrity": "sha512-k8P17w7Eho5Y4l3tZrYxqQdffkI4xwtl8GCxkZs+JdMWZhyrLLlozqWkKLaWrCSlEYQOeIhEnQLhqQgYYU86Rw==", + "license": "Apache-2.0", "dependencies": { - "json-schema-to-ts": "^3.1.1", - "standardwebhooks": "^1.0.0" + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27", + "@vercel/oidc": "3.2.0" }, - "bin": { - "anthropic-ai-sdk": "bin/cli" + "engines": { + "node": ">=18" }, "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.10.tgz", + "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } + "engines": { + "node": ">=18" } }, - "node_modules/@babel/runtime": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", - "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", - "license": "MIT", + "node_modules/@ai-sdk/provider-utils": { + "version": "4.0.27", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.27.tgz", + "integrity": "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8" + }, "engines": { - "node": ">=6.9.0" + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" } }, "node_modules/@clerk/backend": { @@ -227,6 +244,28 @@ "ipaddr.js": "^2.1.0" } }, + "node_modules/@openrouter/ai-sdk-provider": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@openrouter/ai-sdk-provider/-/ai-sdk-provider-2.9.0.tgz", + "integrity": "sha512-Seva+NCa0WUQnJIUE5GzHsUv1WTIeyqwz0ELl2VtS6NP+eF+77yCXGFVOMbvoCM7QMjlnhv7931e89R+8pJdcQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "ai": "^6.0.0", + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", @@ -239,6 +278,12 @@ "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", "license": "MIT" }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, "node_modules/@tanstack/query-core": { "version": "5.100.11", "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.100.11.tgz", @@ -259,12 +304,39 @@ "undici-types": "~6.21.0" } }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, "node_modules/abstract-logging": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==", "license": "MIT" }, + "node_modules/ai": { + "version": "6.0.185", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.185.tgz", + "integrity": "sha512-oGsqscREaTlo75KHZLtwZxRyI+ZBwHV2wRX9B8smHjgOs13WwoCvUyr5aPUWpIBRz406wmIKy1RzoUEq0/WKJw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "3.0.116", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27", + "@opentelemetry/api": "^1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -856,6 +928,15 @@ "url": "https://dotenvx.com" } }, + "node_modules/eventsource-parser": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.8.tgz", + "integrity": "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -1034,6 +1115,12 @@ "node": ">=14" } }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, "node_modules/json-schema-ref-resolver": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz", @@ -1053,19 +1140,6 @@ "dequal": "^2.0.3" } }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -1361,12 +1435,6 @@ "node": ">=12" } }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "license": "MIT" - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -1923,7 +1991,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/backend/package.json b/backend/package.json index cbba04b..60f038a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,12 +10,13 @@ "infer-schema": "tsx src/cli.ts" }, "dependencies": { - "@anthropic-ai/sdk": "^0.97.1", "@clerk/backend": "^3.4.11", + "@openrouter/ai-sdk-provider": "^2.9.0", "@fastify/cors": "^11.0.0", "convex": "^1.39.1", "dotenv": "^16.4.0", "fastify": "^5.0.0", + "ai": "^6.0.0", "fastify-plugin": "^5.1.0", "zod": "^4.4.3" }, diff --git a/backend/prompts/schema-inference.txt b/backend/prompts/schema-inference.txt index ff2fc77..136c293 100644 --- a/backend/prompts/schema-inference.txt +++ b/backend/prompts/schema-inference.txt @@ -1,4 +1,4 @@ -You are a data engineering assistant that converts natural-language prompts into structured dataset schemas. Given a user prompt describing a dataset they want to build, you emit a single tool call to `emit_dataset_schema` with a precise schema definition. +You are a data engineering assistant that converts natural-language prompts into structured dataset schemas. Given a user prompt describing a dataset they want to build, you produce a precise schema definition. Your job is to: @@ -14,7 +14,6 @@ Your job is to: Rules: -- Return ONLY the tool call. Do not include any prose response. - `dataset_name` must be snake_case. - All column `name` values must be snake_case and unique. - Prefer concrete column choices over speculative ones — better to omit a column than guess wildly. diff --git a/backend/src/env.ts b/backend/src/env.ts index 606ab33..7522d7c 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -23,5 +23,5 @@ export const env = { // added. CLERK_SECRET_KEY: process.env.CLERK_SECRET_KEY, - ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, + OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY, }; diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts index 6399cff..070f695 100644 --- a/backend/src/pipeline/schema-inference.ts +++ b/backend/src/pipeline/schema-inference.ts @@ -1,132 +1,48 @@ import { readFileSync } from "node:fs"; -import Anthropic from "@anthropic-ai/sdk"; -import { z } from "zod"; +import { generateText, Output, NoObjectGeneratedError } from "ai"; +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { env } from "../env.js"; import { datasetSchemaSchema, type DatasetSchema } from "./types.js"; -const MODEL = "claude-sonnet-4-6"; -const TOOL_NAME = "emit_dataset_schema"; - const SYSTEM_PROMPT = readFileSync( new URL("../../prompts/schema-inference.txt", import.meta.url), "utf8", ); -if (!env.ANTHROPIC_API_KEY) { - throw new Error("Missing required environment variable: ANTHROPIC_API_KEY"); +function getModel() { + if (!env.OPENROUTER_API_KEY) { + throw new Error("Missing required environment variable: OPENROUTER_API_KEY"); + } + const openrouter = createOpenRouter({ apiKey: env.OPENROUTER_API_KEY }); + return openrouter("anthropic/claude-sonnet-4-6"); } -const client = new Anthropic({ apiKey: env.ANTHROPIC_API_KEY }); - -const datasetSchemaInputSchema = { - type: "object", - required: [ - "dataset_name", - "description", - "columns", - "primary_key", - "retrieval_strategy", - "source_hint", - ], - properties: { - dataset_name: { - type: "string", - description: "snake_case identifier for the dataset", - }, - description: { - type: "string", - description: "Human-readable summary of what this dataset captures", - }, - primary_key: { - type: "string", - description: "Name of the column that uniquely identifies each row", - }, - retrieval_strategy: { - type: "string", - enum: ["search_fetch", "browser", "hybrid"], - description: - "search_fetch for static pages, browser for SPAs/paginated UIs, hybrid if unclear", - }, - source_hint: { - type: "string", - description: - "Specific URL where the data can be found, or a precise description if no single URL exists", - }, - columns: { - type: "array", - minItems: 1, - items: { - type: "object", - required: [ - "name", - "display_name", - "type", - "is_primary_key", - "is_enumerable", - "retrieval_hint", - "nullable", - ], - properties: { - name: { type: "string", description: "snake_case column name" }, - display_name: { type: "string" }, - type: { - type: "string", - enum: ["string", "url", "date", "number", "boolean", "enum"], - }, - is_primary_key: { type: "boolean" }, - is_enumerable: { - type: "boolean", - description: - "true only on the primary key and any other columns whose values can be used to enumerate all rows", - }, - retrieval_hint: { - type: "string", - description: "Where/how to find this column's value in Phase 3", - }, - nullable: { type: "boolean" }, - }, - }, - }, - }, -}; - export async function inferSchema(prompt: string): Promise { - const first = await callOnce(prompt); - const firstParsed = datasetSchemaSchema.safeParse(first); - if (firstParsed.success) return firstParsed.data; - - const errorText = z.prettifyError(firstParsed.error); - const retryMessage = `${prompt}\n\nYour previous output failed validation:\n${errorText}\n\nReturn a corrected DatasetSchema.`; - const second = await callOnce(retryMessage); - return datasetSchemaSchema.parse(second); + const model = getModel(); + try { + return await callOnce(model, prompt); + } catch (error) { + if (NoObjectGeneratedError.isInstance(error)) { + const detail = error.cause ? String(error.cause) : error.text; + const retry = `${prompt}\n\nYour previous output failed validation:\n${detail}\n\nReturn a corrected DatasetSchema.`; + return await callOnce(model, retry); + } + throw error; + } } -async function callOnce(userMessage: string): Promise { - const response = await client.messages.create({ - model: MODEL, - max_tokens: 4096, - system: [ - { - type: "text", - text: SYSTEM_PROMPT, - cache_control: { type: "ephemeral" }, - }, - ], - tools: [ - { - name: TOOL_NAME, - description: - "Emit the inferred DatasetSchema describing the dataset to be built.", - input_schema: datasetSchemaInputSchema as Anthropic.Tool["input_schema"], - }, - ], - tool_choice: { type: "tool", name: TOOL_NAME }, - messages: [{ role: "user", content: userMessage }], +async function callOnce( + model: Parameters[0]["model"], + prompt: string, +): Promise { + const { output } = await generateText({ + model, + output: Output.object({ schema: datasetSchemaSchema }), + system: SYSTEM_PROMPT, + maxTokens: 4096, + prompt, }); - - for (const block of response.content) { - if (block.type === "tool_use") return block.input; - } - throw new Error("Model did not emit a tool_use block"); + if (!output) throw new Error("Model did not generate a valid schema object"); + return output; } From f54e368df25de150b839df17c641e75588a1f415 Mon Sep 17 00:00:00 2001 From: Simantak Dabhade Date: Wed, 20 May 2026 13:58:05 -0700 Subject: [PATCH 4/4] Add MMeteorL's review comments to schema-inference prompt Preserves suggestions about replacing retrieval_strategy/source_hint with search_queries for future implementation. Co-Authored-By: Claude Opus 4.6 --- backend/prompts/schema-inference.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/prompts/schema-inference.txt b/backend/prompts/schema-inference.txt index 136c293..9752429 100644 --- a/backend/prompts/schema-inference.txt +++ b/backend/prompts/schema-inference.txt @@ -17,3 +17,20 @@ Rules: - `dataset_name` must be snake_case. - All column `name` values must be snake_case and unique. - Prefer concrete column choices over speculative ones — better to omit a column than guess wildly. + +# @MMeteorL's comments/suggestions: +# This may be too early in the agent workflow to suggest these without more +# context. In the current agent system, the agent would first use Tinyfish +# search to search for candidate urls and fetch those results for analysis +# of what is the best way to retrieve. +# +# For the same reason, instead of source_hint, it may be helpful to generate +# search_query_hint. +# +# Retrieval_hint is a great idea, however, because of the varied sources +# that Tinyfish search would propose, this may not be the best fit. This +# should be a decision left for downstream agent with more information. +# +# My recommendation is to produce: 'search_queries': generate [X number] of +# search queries based on the user prompt and data spec. These search queries +# should be differentiated from each other.