diff --git a/.gitignore b/.gitignore index f21d121..f75ce72 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ yarn-debug.log* *.bak tmp/ temp/ + +backend/output/ diff --git a/backend/.env.example b/backend/.env.example index beece7e..adfa070 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -9,3 +9,7 @@ CONVEX_SELF_HOSTED_ADMIN_KEY= # Required once any user-facing protected route is added. # Same value as the frontend's CLERK_SECRET_KEY. CLERK_SECRET_KEY= + +# OpenRouter API key — required by the schema-inference CLI (npm run infer-schema). +# Generate at https://openrouter.ai/settings/keys +OPENROUTER_API_KEY=sk-or-... diff --git a/backend/package-lock.json b/backend/package-lock.json index 7931616..730bd2f 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -10,10 +10,13 @@ "dependencies": { "@clerk/backend": "^3.4.11", "@fastify/cors": "^11.0.0", + "@openrouter/ai-sdk-provider": "^2.9.0", + "ai": "^6.0.0", "convex": "^1.39.1", "dotenv": "^16.4.0", "fastify": "^5.0.0", - "fastify-plugin": "^5.1.0" + "fastify-plugin": "^5.1.0", + "zod": "^4.4.3" }, "devDependencies": { "@types/node": "^22.0.0", @@ -21,6 +24,52 @@ "typescript": "^5.0.0" } }, + "node_modules/@ai-sdk/gateway": { + "version": "3.0.116", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.116.tgz", + "integrity": "sha512-k8P17w7Eho5Y4l3tZrYxqQdffkI4xwtl8GCxkZs+JdMWZhyrLLlozqWkKLaWrCSlEYQOeIhEnQLhqQgYYU86Rw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.10.tgz", + "integrity": "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "4.0.27", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.27.tgz", + "integrity": "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.10", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@clerk/backend": { "version": "3.4.11", "resolved": "https://registry.npmjs.org/@clerk/backend/-/backend-3.4.11.tgz", @@ -195,6 +244,28 @@ "ipaddr.js": "^2.1.0" } }, + "node_modules/@openrouter/ai-sdk-provider": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@openrouter/ai-sdk-provider/-/ai-sdk-provider-2.9.0.tgz", + "integrity": "sha512-Seva+NCa0WUQnJIUE5GzHsUv1WTIeyqwz0ELl2VtS6NP+eF+77yCXGFVOMbvoCM7QMjlnhv7931e89R+8pJdcQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "ai": "^6.0.0", + "zod": "^3.25.0 || ^4.0.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", @@ -207,6 +278,12 @@ "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", "license": "MIT" }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, "node_modules/@tanstack/query-core": { "version": "5.100.11", "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.100.11.tgz", @@ -227,12 +304,39 @@ "undici-types": "~6.21.0" } }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, "node_modules/abstract-logging": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", "integrity": "sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==", "license": "MIT" }, + "node_modules/ai": { + "version": "6.0.185", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.185.tgz", + "integrity": "sha512-oGsqscREaTlo75KHZLtwZxRyI+ZBwHV2wRX9B8smHjgOs13WwoCvUyr5aPUWpIBRz406wmIKy1RzoUEq0/WKJw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "3.0.116", + "@ai-sdk/provider": "3.0.10", + "@ai-sdk/provider-utils": "4.0.27", + "@opentelemetry/api": "^1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -824,6 +928,15 @@ "url": "https://dotenvx.com" } }, + "node_modules/eventsource-parser": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.8.tgz", + "integrity": "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -1002,6 +1115,12 @@ "node": ">=14" } }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, "node_modules/json-schema-ref-resolver": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-schema-ref-resolver/-/json-schema-ref-resolver-3.0.0.tgz", @@ -1866,6 +1985,15 @@ "optional": true } } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/backend/package.json b/backend/package.json index eb61b32..60f038a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -6,15 +6,19 @@ "scripts": { "dev": "tsx watch src/index.ts", "build": "tsc", - "start": "node dist/index.js" + "start": "node dist/index.js", + "infer-schema": "tsx src/cli.ts" }, "dependencies": { "@clerk/backend": "^3.4.11", + "@openrouter/ai-sdk-provider": "^2.9.0", "@fastify/cors": "^11.0.0", "convex": "^1.39.1", "dotenv": "^16.4.0", "fastify": "^5.0.0", - "fastify-plugin": "^5.1.0" + "ai": "^6.0.0", + "fastify-plugin": "^5.1.0", + "zod": "^4.4.3" }, "devDependencies": { "@types/node": "^22.0.0", diff --git a/backend/prompts/schema-inference.txt b/backend/prompts/schema-inference.txt new file mode 100644 index 0000000..9752429 --- /dev/null +++ b/backend/prompts/schema-inference.txt @@ -0,0 +1,36 @@ +You are a data engineering assistant that converts natural-language prompts into structured dataset schemas. Given a user prompt describing a dataset they want to build, you produce a precise schema definition. + +Your job is to: + +1. Identify the universe of entities the user wants to collect. Each entity becomes one row in the dataset. +2. Pick a clear primary key — the column whose values uniquely identify each row. This is usually a name, ID, or canonical URL. Exactly one column must have `is_primary_key: true`, and its `name` must equal `primary_key`. The primary key column must have `nullable: false` and `is_enumerable: true`. +3. Choose useful columns. Each column captures one fact about the entity. Use snake_case names. Mark `is_enumerable: true` only on columns whose values can be used to list all rows (typically just the primary key, and occasionally one or two others when a source page lists them alongside the primary key). +4. Set `retrieval_strategy`: + - `search_fetch` — the data lives on a static page or sitemap that can be fetched as HTML. + - `browser` — the source is a JavaScript-heavy SPA, requires scroll/click to reveal items, or paginates client-side. + - `hybrid` — unclear; the pipeline will try search_fetch first and fall back to browser. +5. Set `source_hint` to a specific URL whenever possible (e.g. `https://www.ycombinator.com/companies?industry=Fintech`). Avoid vague descriptions. +6. Write a `retrieval_hint` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row. + +Rules: + +- `dataset_name` must be snake_case. +- All column `name` values must be snake_case and unique. +- Prefer concrete column choices over speculative ones — better to omit a column than guess wildly. + +# @MMeteorL's comments/suggestions: +# This may be too early in the agent workflow to suggest these without more +# context. In the current agent system, the agent would first use Tinyfish +# search to search for candidate urls and fetch those results for analysis +# of what is the best way to retrieve. +# +# For the same reason, instead of source_hint, it may be helpful to generate +# search_query_hint. +# +# Retrieval_hint is a great idea, however, because of the varied sources +# that Tinyfish search would propose, this may not be the best fit. This +# should be a decision left for downstream agent with more information. +# +# My recommendation is to produce: 'search_queries': generate [X number] of +# search queries based on the user prompt and data spec. These search queries +# should be differentiated from each other. diff --git a/backend/src/cli.ts b/backend/src/cli.ts new file mode 100644 index 0000000..0d14680 --- /dev/null +++ b/backend/src/cli.ts @@ -0,0 +1,42 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { randomBytes } from "node:crypto"; +import { join } from "node:path"; + +import { inferSchema } from "./pipeline/schema-inference.js"; + +function parsePrompt(argv: string[]): string { + const idx = argv.findIndex((a) => a === "--prompt"); + if (idx === -1 || idx === argv.length - 1) { + throw new Error('Usage: npm run infer-schema -- --prompt ""'); + } + const value = argv[idx + 1]; + if (!value.trim()) throw new Error("--prompt requires a non-empty value"); + return value; +} + +function generateRunId(): string { + const ts = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14); + const rand = randomBytes(3).toString("hex"); + return `${ts}-${rand}`; +} + +async function main() { + const prompt = parsePrompt(process.argv.slice(2)); + const runId = generateRunId(); + const outDir = join("output", runId); + + console.log(`Inferring schema for: "${prompt}"`); + const schema = await inferSchema(prompt); + + mkdirSync(outDir, { recursive: true }); + const schemaPath = join(outDir, "schema.json"); + writeFileSync(schemaPath, JSON.stringify(schema, null, 2) + "\n"); + + console.log(`Run ID: ${runId}`); + console.log(`Schema: backend/${schemaPath}`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/backend/src/env.ts b/backend/src/env.ts index 62e3ce9..7522d7c 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -22,4 +22,6 @@ export const env = { // today because no protected routes exist yet; required as soon as one is // added. CLERK_SECRET_KEY: process.env.CLERK_SECRET_KEY, + + OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY, }; diff --git a/backend/src/pipeline/schema-inference.ts b/backend/src/pipeline/schema-inference.ts new file mode 100644 index 0000000..070f695 --- /dev/null +++ b/backend/src/pipeline/schema-inference.ts @@ -0,0 +1,48 @@ +import { readFileSync } from "node:fs"; +import { generateText, Output, NoObjectGeneratedError } from "ai"; +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; + +import { env } from "../env.js"; +import { datasetSchemaSchema, type DatasetSchema } from "./types.js"; + +const SYSTEM_PROMPT = readFileSync( + new URL("../../prompts/schema-inference.txt", import.meta.url), + "utf8", +); + +function getModel() { + if (!env.OPENROUTER_API_KEY) { + throw new Error("Missing required environment variable: OPENROUTER_API_KEY"); + } + const openrouter = createOpenRouter({ apiKey: env.OPENROUTER_API_KEY }); + return openrouter("anthropic/claude-sonnet-4-6"); +} + +export async function inferSchema(prompt: string): Promise { + const model = getModel(); + try { + return await callOnce(model, prompt); + } catch (error) { + if (NoObjectGeneratedError.isInstance(error)) { + const detail = error.cause ? String(error.cause) : error.text; + const retry = `${prompt}\n\nYour previous output failed validation:\n${detail}\n\nReturn a corrected DatasetSchema.`; + return await callOnce(model, retry); + } + throw error; + } +} + +async function callOnce( + model: Parameters[0]["model"], + prompt: string, +): Promise { + const { output } = await generateText({ + model, + output: Output.object({ schema: datasetSchemaSchema }), + system: SYSTEM_PROMPT, + maxTokens: 4096, + prompt, + }); + if (!output) throw new Error("Model did not generate a valid schema object"); + return output; +} diff --git a/backend/src/pipeline/types.ts b/backend/src/pipeline/types.ts new file mode 100644 index 0000000..2d5cc58 --- /dev/null +++ b/backend/src/pipeline/types.ts @@ -0,0 +1,116 @@ +import { z } from "zod"; + +export const columnTypeSchema = z.enum([ + "string", + "url", + "date", + "number", + "boolean", + "enum", +]); +export type ColumnType = z.infer; + +export const retrievalStrategySchema = z.enum([ + "search_fetch", + "browser", + "hybrid", +]); +export type RetrievalStrategy = z.infer; + +const snakeCase = /^[a-z][a-z0-9_]*$/; + +export const columnDefinitionSchema = z.object({ + name: z.string().regex(snakeCase, "must be snake_case"), + display_name: z.string().min(1), + type: columnTypeSchema, + is_primary_key: z.boolean(), + is_enumerable: z.boolean(), + retrieval_hint: z.string(), + nullable: z.boolean(), +}); +export type ColumnDefinition = z.infer; + +export const datasetSchemaSchema = z + .object({ + dataset_name: z.string().regex(snakeCase, "must be snake_case"), + description: z.string().min(1), + columns: z.array(columnDefinitionSchema).min(1), + primary_key: z.string(), + retrieval_strategy: retrievalStrategySchema, + source_hint: z.string().min(1), + }) + .superRefine((data, ctx) => { + const names = data.columns.map((c) => c.name); + const dupes = [...new Set(names.filter((n, i) => names.indexOf(n) !== i))]; + if (dupes.length > 0) { + ctx.addIssue({ + code: "custom", + message: `duplicate column names: ${dupes.join(", ")}`, + path: ["columns"], + }); + } + + const pkCols = data.columns.filter((c) => c.is_primary_key); + if (pkCols.length !== 1) { + ctx.addIssue({ + code: "custom", + message: `exactly one column must have is_primary_key=true (found ${pkCols.length})`, + path: ["columns"], + }); + return; + } + + const pk = pkCols[0]; + if (pk.name !== data.primary_key) { + ctx.addIssue({ + code: "custom", + message: `primary_key '${data.primary_key}' does not match the column flagged is_primary_key ('${pk.name}')`, + path: ["primary_key"], + }); + } + if (pk.nullable) { + ctx.addIssue({ + code: "custom", + message: "primary key column must not be nullable", + path: ["columns"], + }); + } + if (!pk.is_enumerable) { + ctx.addIssue({ + code: "custom", + message: "primary key column must have is_enumerable=true", + path: ["columns"], + }); + } + }); +export type DatasetSchema = z.infer; + +export const datasetRowValueSchema = z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), +]); +export type DatasetRowValue = z.infer; + +export const datasetRowSchema = z.record(z.string(), datasetRowValueSchema); +export type DatasetRow = z.infer; + +export const endpointCallSchema = z.object({ + endpoint: z.enum(["search", "fetch", "browser"]), + count: z.number().int().nonnegative(), +}); +export type EndpointCall = z.infer; + +export const runManifestSchema = z.object({ + run_id: z.string(), + prompt: z.string(), + schema_path: z.string(), + dataset_path: z.string(), + csv_path: z.string(), + row_count: z.number().int().nonnegative(), + columns_filled: z.array(z.string()), + created_at: z.string(), + endpoints_called: z.array(endpointCallSchema), +}); +export type RunManifest = z.infer;