Prototype dataset builder harness

.gitignore

@@ -1,5 +1,11 @@
 .DS_Store
 node_modules/
+backend/.npm-cache/
 .env
 .env.local
 Project_BigSet_brief.md
+notion-docs.md
+image.png
+image-1.png
+.omx/
+/car

backend/.env.example

@@ -4,3 +4,10 @@ BETTER_AUTH_URL=http://localhost:3501
 CLIENT_ORIGIN=http://localhost:3500
 DATABASE_URL=postgres://bigset:bigset@localhost:5432/bigset
 PORT=3501
+
+# Optional. Dataset-builder planner uses deterministic fallback when missing.
+OPENROUTER_API_KEY=""
+OPENROUTER_MODEL=openai/gpt-4.1-mini
+
+# Optional. Local TinyFish CLI harness uses this when executing search/fetch/agent runs.
+TINYFISH_API_KEY=""

backend/drizzle.config.ts

@@ -1,6 +1,10 @@
-import "dotenv/config";
+import { config } from "dotenv";
 import { defineConfig } from "drizzle-kit";
 
+config({ path: ".env.local" });
+config({ path: "../.env.local" });
+config();
+
 const databaseUrl = process.env.DATABASE_URL;
 if (!databaseUrl) {
   throw new Error("DATABASE_URL is required for Drizzle Kit");

backend/package.json

@@ -5,6 +5,7 @@
   "private": true,
   "scripts": {
     "dev": "tsx watch src/index.ts",
+    "builder:plan": "tsx src/dataset-builder/run-prototype.ts",
     "build": "tsc",
     "start": "node dist/index.js",
     "db:push": "drizzle-kit push"

backend/src/dataset-builder/agent-harness.ts

@@ -0,0 +1,136 @@
+import type {
+  AgentHarnessStage,
+  DatasetBuildPlan,
+  DatasetColumnDefinition,
+  DatasetSourceStrategy,
+} from "./types.js";
+
+export function createHarnessStages(
+  sourceStrategy: DatasetSourceStrategy,
+  hasMissingInputs: boolean
+): AgentHarnessStage[] {
+  const stages: AgentHarnessStage[] = [];
+
+  if (hasMissingInputs) {
+    stages.push({
+      id: "clarify-missing-inputs",
+      title: "Ask for missing inputs",
+      purpose:
+        "Pause before scraping when the target sites need user-specific values.",
+      tool: "user_input",
+      canRunWithoutUser: false,
+    });
+  }
+
+  stages.push(
+    {
+      id: "discover-source-candidates",
+      title: "Discover source candidates",
+      purpose: "Use TinyFish Search to find source pages likely to contain rows.",
+      tool: "tinyfish_search",
+      canRunWithoutUser: true,
+    },
+    {
+      id: "fetch-source-pages",
+      title: "Fetch clean source content",
+      purpose: "Use TinyFish Fetch before browser automation for speed and cost.",
+      tool: "tinyfish_fetch",
+      canRunWithoutUser: true,
+    }
+  );
+
+  if (sourceStrategy !== "search_fetch") {
+    stages.push({
+      id: "browser-deep-extraction",
+      title: "Deep browser extraction",
+      purpose:
+        "Use TinyFish Agent or browser automation only when search/fetch cannot reach the value.",
+      tool: "tinyfish_agent",
+      canRunWithoutUser: !hasMissingInputs,
+    });
+  }
+
+  stages.push(
+    {
+      id: "validate-cells",
+      title: "Validate extracted cells",
+      purpose:
+        "Reject rows that do not match schema, keep source URLs, and mark missing cells.",
+      tool: "validator",
+      canRunWithoutUser: true,
+    },
+    {
+      id: "upsert-rows",
+      title: "Replace changed row values",
+      purpose:
+        "Use the identity column to replace current values instead of appending duplicates.",
+      tool: "database",
+      canRunWithoutUser: true,
+    }
+  );
+
+  return stages;
+}
+
+export function createTinyFishAgentGoal(plan: DatasetBuildPlan): string {
+  const columnDescriptions = plan.schema.columns
+    .map((column) => `- ${column.name}: ${column.description}`)
+    .join("\n");
+
+  return [
+    `Build rows for dataset: ${plan.datasetName}.`,
+    `User request: ${plan.userRequest}`,
+    `Identity column: ${plan.schema.identityColumnName}`,
+    "Return only rows backed by source URLs.",
+    "Mark missing fields as null instead of guessing.",
+    "Columns:",
+    columnDescriptions,
+  ].join("\n");
+}
+
+export function createTinyFishAgentOutputSchema(plan: DatasetBuildPlan) {
+  const rowProperties = Object.fromEntries(
+    plan.schema.columns.map((column) => [column.name, jsonSchemaForColumn(column)])
+  );
+
+  return {
+    type: "object",
+    additionalProperties: false,
+    required: ["rows", "sourceUrls", "validationIssues"],
+    properties: {
+      rows: {
+        type: "array",
+        items: {
+          type: "object",
+          additionalProperties: false,
+          required: [plan.schema.identityColumnName],
+          properties: rowProperties,
+        },
+      },
+      sourceUrls: {
+        type: "array",
+        items: { type: "string" },
+      },
+      validationIssues: {
+        type: "array",
+        items: { type: "string" },
+      },
+    },
+  };
+}
+
+function jsonSchemaForColumn(column: DatasetColumnDefinition) {
+  if (column.kind === "number") {
+    return { type: ["number", "null"], description: column.description };
+  }
+  if (column.kind === "boolean") {
+    return { type: ["boolean", "null"], description: column.description };
+  }
+  if (column.kind === "json") {
+    return {
+      type: ["object", "array", "null"],
+      description: column.description,
+    };
+  }
+  return { type: ["string", "null"], description: column.description };
+}