Add Phase 1 schema-inference CLI (prompt → DatasetSchema)

.gitignore

@@ -19,3 +19,5 @@ yarn-debug.log*
 *.bak
 tmp/
 temp/
+
+backend/output/

backend/.env.example

@@ -9,3 +9,7 @@ CONVEX_SELF_HOSTED_ADMIN_KEY=
 # Required once any user-facing protected route is added.
 # Same value as the frontend's CLERK_SECRET_KEY.
 CLERK_SECRET_KEY=
+
+# OpenRouter API key — required by the schema-inference CLI (npm run infer-schema).
+# Generate at https://openrouter.ai/settings/keys
+OPENROUTER_API_KEY=sk-or-...

backend/package.json

@@ -6,15 +6,19 @@
   "scripts": {
     "dev": "tsx watch src/index.ts",
     "build": "tsc",
-    "start": "node dist/index.js"
+    "start": "node dist/index.js",
+    "infer-schema": "tsx src/cli.ts"
   },
   "dependencies": {
     "@clerk/backend": "^3.4.11",
+    "@openrouter/ai-sdk-provider": "^2.9.0",
     "@fastify/cors": "^11.0.0",
     "convex": "^1.39.1",
     "dotenv": "^16.4.0",
     "fastify": "^5.0.0",
-    "fastify-plugin": "^5.1.0"
+    "ai": "^6.0.0",
+    "fastify-plugin": "^5.1.0",
+    "zod": "^4.4.3"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",

backend/prompts/schema-inference.txt

@@ -0,0 +1,36 @@
+You are a data engineering assistant that converts natural-language prompts into structured dataset schemas. Given a user prompt describing a dataset they want to build, you produce a precise schema definition.
+
+Your job is to:
+
+1. Identify the universe of entities the user wants to collect. Each entity becomes one row in the dataset.
+2. Pick a clear primary key — the column whose values uniquely identify each row. This is usually a name, ID, or canonical URL. Exactly one column must have `is_primary_key: true`, and its `name` must equal `primary_key`. The primary key column must have `nullable: false` and `is_enumerable: true`.
+3. Choose useful columns. Each column captures one fact about the entity. Use snake_case names. Mark `is_enumerable: true` only on columns whose values can be used to list all rows (typically just the primary key, and occasionally one or two others when a source page lists them alongside the primary key).
+4. Set `retrieval_strategy`:
+   - `search_fetch` — the data lives on a static page or sitemap that can be fetched as HTML.
+   - `browser` — the source is a JavaScript-heavy SPA, requires scroll/click to reveal items, or paginates client-side.
+   - `hybrid` — unclear; the pipeline will try search_fetch first and fall back to browser.
+5. Set `source_hint` to a specific URL whenever possible (e.g. `https://www.ycombinator.com/companies?industry=Fintech`). Avoid vague descriptions.
+6. Write a `retrieval_hint` for each column describing where/how the value can be found later. Downstream agents will use this to fill the column for each row.
+
+Rules:
+
+- `dataset_name` must be snake_case.
+- All column `name` values must be snake_case and unique.
+- Prefer concrete column choices over speculative ones — better to omit a column than guess wildly.
+
+# @MMeteorL's comments/suggestions:
+# This may be too early in the agent workflow to suggest these without more
+# context. In the current agent system, the agent would first use Tinyfish
+# search to search for candidate urls and fetch those results for analysis
+# of what is the best way to retrieve.
+#
+# For the same reason, instead of source_hint, it may be helpful to generate
+# search_query_hint.
+#
+# Retrieval_hint is a great idea, however, because of the varied sources
+# that Tinyfish search would propose, this may not be the best fit. This
+# should be a decision left for downstream agent with more information.
+#
+# My recommendation is to produce: 'search_queries': generate [X number] of
+# search queries based on the user prompt and data spec. These search queries
+# should be differentiated from each other.

backend/src/cli.ts

@@ -0,0 +1,42 @@
+import { mkdirSync, writeFileSync } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { join } from "node:path";
+
+import { inferSchema } from "./pipeline/schema-inference.js";
+
+function parsePrompt(argv: string[]): string {
+  const idx = argv.findIndex((a) => a === "--prompt");
+  if (idx === -1 || idx === argv.length - 1) {
+    throw new Error('Usage: npm run infer-schema -- --prompt "<your prompt>"');
+  }
+  const value = argv[idx + 1];
+  if (!value.trim()) throw new Error("--prompt requires a non-empty value");
+  return value;
+}
+
+function generateRunId(): string {
+  const ts = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14);
+  const rand = randomBytes(3).toString("hex");
+  return `${ts}-${rand}`;
+}
+
+async function main() {
+  const prompt = parsePrompt(process.argv.slice(2));
+  const runId = generateRunId();
+  const outDir = join("output", runId);
+
+  console.log(`Inferring schema for: "${prompt}"`);
+  const schema = await inferSchema(prompt);
+
+  mkdirSync(outDir, { recursive: true });
+  const schemaPath = join(outDir, "schema.json");
+  writeFileSync(schemaPath, JSON.stringify(schema, null, 2) + "\n");
+
+  console.log(`Run ID: ${runId}`);
+  console.log(`Schema: backend/${schemaPath}`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

backend/src/env.ts

@@ -22,4 +22,6 @@ export const env = {
   // today because no protected routes exist yet; required as soon as one is
   // added.
   CLERK_SECRET_KEY: process.env.CLERK_SECRET_KEY,
+
+  OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY,
 };

backend/src/pipeline/schema-inference.ts

@@ -0,0 +1,48 @@
+import { readFileSync } from "node:fs";
+import { generateText, Output, NoObjectGeneratedError } from "ai";
+import { createOpenRouter } from "@openrouter/ai-sdk-provider";
+
+import { env } from "../env.js";
+import { datasetSchemaSchema, type DatasetSchema } from "./types.js";
+
+const SYSTEM_PROMPT = readFileSync(
+  new URL("../../prompts/schema-inference.txt", import.meta.url),
+  "utf8",
+);
+
+function getModel() {
+  if (!env.OPENROUTER_API_KEY) {
+    throw new Error("Missing required environment variable: OPENROUTER_API_KEY");
+  }
+  const openrouter = createOpenRouter({ apiKey: env.OPENROUTER_API_KEY });
+  return openrouter("anthropic/claude-sonnet-4-6");
+}
+
+export async function inferSchema(prompt: string): Promise<DatasetSchema> {
+  const model = getModel();
+  try {
+    return await callOnce(model, prompt);
+  } catch (error) {
+    if (NoObjectGeneratedError.isInstance(error)) {
+      const detail = error.cause ? String(error.cause) : error.text;
+      const retry = `${prompt}\n\nYour previous output failed validation:\n${detail}\n\nReturn a corrected DatasetSchema.`;
+      return await callOnce(model, retry);
+    }
+    throw error;
+  }
+}
+
+async function callOnce(
+  model: Parameters<typeof generateText>[0]["model"],
+  prompt: string,
+): Promise<DatasetSchema> {
+  const { output } = await generateText({
+    model,
+    output: Output.object({ schema: datasetSchemaSchema }),
+    system: SYSTEM_PROMPT,
+    maxTokens: 4096,
+    prompt,
+  });
+  if (!output) throw new Error("Model did not generate a valid schema object");
+  return output;
+}