diff --git a/.gitignore b/.gitignore index 53c1673..5539c4b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,11 @@ .DS_Store node_modules/ +backend/.npm-cache/ .env .env.local Project_BigSet_brief.md +notion-docs.md +image.png +image-1.png +.omx/ +/car diff --git a/backend/.env.example b/backend/.env.example index bfa35d1..86e8567 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -4,3 +4,10 @@ BETTER_AUTH_URL=http://localhost:3501 CLIENT_ORIGIN=http://localhost:3500 DATABASE_URL=postgres://bigset:bigset@localhost:5432/bigset PORT=3501 + +# Optional. Dataset-builder planner uses deterministic fallback when missing. +OPENROUTER_API_KEY="" +OPENROUTER_MODEL=openai/gpt-4.1-mini + +# Optional. Local TinyFish CLI harness uses this when executing search/fetch/agent runs. +TINYFISH_API_KEY="" diff --git a/backend/drizzle.config.ts b/backend/drizzle.config.ts index f3345b9..83edec7 100644 --- a/backend/drizzle.config.ts +++ b/backend/drizzle.config.ts @@ -1,6 +1,10 @@ -import "dotenv/config"; +import { config } from "dotenv"; import { defineConfig } from "drizzle-kit"; +config({ path: ".env.local" }); +config({ path: "../.env.local" }); +config(); + const databaseUrl = process.env.DATABASE_URL; if (!databaseUrl) { throw new Error("DATABASE_URL is required for Drizzle Kit"); diff --git a/backend/package.json b/backend/package.json index f4581a2..e27482b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -5,6 +5,7 @@ "private": true, "scripts": { "dev": "tsx watch src/index.ts", + "builder:plan": "tsx src/dataset-builder/run-prototype.ts", "build": "tsc", "start": "node dist/index.js", "db:push": "drizzle-kit push" diff --git a/backend/src/dataset-builder/agent-harness.ts b/backend/src/dataset-builder/agent-harness.ts new file mode 100644 index 0000000..5b9f90a --- /dev/null +++ b/backend/src/dataset-builder/agent-harness.ts @@ -0,0 +1,136 @@ +import type { + AgentHarnessStage, + DatasetBuildPlan, + DatasetColumnDefinition, + DatasetSourceStrategy, +} from "./types.js"; + +export function createHarnessStages( + sourceStrategy: DatasetSourceStrategy, + hasMissingInputs: boolean +): AgentHarnessStage[] { + const stages: AgentHarnessStage[] = []; + + if (hasMissingInputs) { + stages.push({ + id: "clarify-missing-inputs", + title: "Ask for missing inputs", + purpose: + "Pause before scraping when the target sites need user-specific values.", + tool: "user_input", + canRunWithoutUser: false, + }); + } + + stages.push( + { + id: "discover-source-candidates", + title: "Discover source candidates", + purpose: "Use TinyFish Search to find source pages likely to contain rows.", + tool: "tinyfish_search", + canRunWithoutUser: true, + }, + { + id: "fetch-source-pages", + title: "Fetch clean source content", + purpose: "Use TinyFish Fetch before browser automation for speed and cost.", + tool: "tinyfish_fetch", + canRunWithoutUser: true, + } + ); + + if (sourceStrategy !== "search_fetch") { + stages.push({ + id: "browser-deep-extraction", + title: "Deep browser extraction", + purpose: + "Use TinyFish Agent or browser automation only when search/fetch cannot reach the value.", + tool: "tinyfish_agent", + canRunWithoutUser: !hasMissingInputs, + }); + } + + stages.push( + { + id: "validate-cells", + title: "Validate extracted cells", + purpose: + "Reject rows that do not match schema, keep source URLs, and mark missing cells.", + tool: "validator", + canRunWithoutUser: true, + }, + { + id: "upsert-rows", + title: "Replace changed row values", + purpose: + "Use the identity column to replace current values instead of appending duplicates.", + tool: "database", + canRunWithoutUser: true, + } + ); + + return stages; +} + +export function createTinyFishAgentGoal(plan: DatasetBuildPlan): string { + const columnDescriptions = plan.schema.columns + .map((column) => `- ${column.name}: ${column.description}`) + .join("\n"); + + return [ + `Build rows for dataset: ${plan.datasetName}.`, + `User request: ${plan.userRequest}`, + `Identity column: ${plan.schema.identityColumnName}`, + "Return only rows backed by source URLs.", + "Mark missing fields as null instead of guessing.", + "Columns:", + columnDescriptions, + ].join("\n"); +} + +export function createTinyFishAgentOutputSchema(plan: DatasetBuildPlan) { + const rowProperties = Object.fromEntries( + plan.schema.columns.map((column) => [column.name, jsonSchemaForColumn(column)]) + ); + + return { + type: "object", + additionalProperties: false, + required: ["rows", "sourceUrls", "validationIssues"], + properties: { + rows: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: [plan.schema.identityColumnName], + properties: rowProperties, + }, + }, + sourceUrls: { + type: "array", + items: { type: "string" }, + }, + validationIssues: { + type: "array", + items: { type: "string" }, + }, + }, + }; +} + +function jsonSchemaForColumn(column: DatasetColumnDefinition) { + if (column.kind === "number") { + return { type: ["number", "null"], description: column.description }; + } + if (column.kind === "boolean") { + return { type: ["boolean", "null"], description: column.description }; + } + if (column.kind === "json") { + return { + type: ["object", "array", "null"], + description: column.description, + }; + } + return { type: ["string", "null"], description: column.description }; +} diff --git a/backend/src/dataset-builder/openrouter.ts b/backend/src/dataset-builder/openrouter.ts new file mode 100644 index 0000000..7477f3d --- /dev/null +++ b/backend/src/dataset-builder/openrouter.ts @@ -0,0 +1,366 @@ +import type { + AgentHarnessStage, + ClarifyingQuestion, + DatasetBuildPlan, + DatasetBuildRequest, + DatasetColumnDefinition, + DatasetSchema, + DatasetSourceStrategy, + DatasetUpdateCadence, +} from "./types.js"; + +interface OpenRouterChoice { + message?: { + content?: string; + }; +} + +interface OpenRouterChatResponse { + choices?: OpenRouterChoice[]; +} + +interface OpenRouterPlannerConfig { + apiKey: string; + model: string; +} + +export class OpenRouterPlannerClient { + private readonly apiKey: string; + private readonly model: string; + + constructor(config: OpenRouterPlannerConfig) { + this.apiKey = config.apiKey; + this.model = config.model; + } + + async refineDatasetBuildPlan( + request: DatasetBuildRequest, + draftPlan: DatasetBuildPlan + ): Promise { + const abortController = new AbortController(); + const timeout = setTimeout(() => abortController.abort(), 30_000); + + const response = await fetch( + "https://openrouter.ai/api/v1/chat/completions", + { + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey}`, + "Content-Type": "application/json", + "HTTP-Referer": "https://github.com/tinyfish-io/bigset", + "X-Title": "BigSet Dataset Builder", + }, + signal: abortController.signal, + body: JSON.stringify({ + model: this.model, + response_format: { type: "json_object" }, + messages: [ + { + role: "system", + content: + "You design reliable web-data agent harnesses. Return only JSON. Preserve the input shape exactly. Never invent user-provided facts.", + }, + { + role: "user", + content: JSON.stringify({ + task: + "Refine this BigSet dataset build plan. Improve schema, clarifying questions, source strategy, validation rules, and next actions.", + request, + draftPlan, + }), + }, + ], + }), + } + ).finally(() => { + clearTimeout(timeout); + }); + + if (!response.ok) { + throw new Error(`OpenRouter returned ${response.status}`); + } + + const payload = (await response.json()) as OpenRouterChatResponse; + const content = payload.choices?.[0]?.message?.content; + if (!content) { + throw new Error("OpenRouter returned no message content."); + } + + return normalizeOpenRouterPlan(content, draftPlan); + } +} + +export function createOpenRouterPlannerClient(config: { + apiKey?: string; + model: string; +}): OpenRouterPlannerClient | undefined { + if (!config.apiKey) { + return undefined; + } + + return new OpenRouterPlannerClient({ + apiKey: config.apiKey, + model: config.model, + }); +} + +function normalizeOpenRouterPlan( + rawContent: string, + fallbackPlan: DatasetBuildPlan +): DatasetBuildPlan { + const parsedPlan = JSON.parse(extractJsonObject(rawContent)) as Record< + string, + unknown + >; + + const schema = normalizeSchema(parsedPlan.schema, fallbackPlan.schema); + if (!schema.columns.length) { + throw new Error("OpenRouter plan did not include schema columns."); + } + + return { + ...fallbackPlan, + datasetName: stringValue(parsedPlan.datasetName) ?? fallbackPlan.datasetName, + userRequest: fallbackPlan.userRequest, + updateCadence: + updateCadenceValue(parsedPlan.updateCadence) ?? fallbackPlan.updateCadence, + schema, + sourceStrategy: + sourceStrategyValue(parsedPlan.sourceStrategy) ?? + fallbackPlan.sourceStrategy, + clarifyingQuestions: normalizeClarifyingQuestions( + parsedPlan.clarifyingQuestions, + fallbackPlan.clarifyingQuestions + ), + harnessStages: normalizeHarnessStages( + parsedPlan.harnessStages, + fallbackPlan.harnessStages + ), + validationRules: + stringArrayValue(parsedPlan.validationRules) ?? + fallbackPlan.validationRules, + replacementPolicy: + stringValue(parsedPlan.replacementPolicy) ?? + fallbackPlan.replacementPolicy, + nextActions: + stringArrayValue(parsedPlan.nextActions) ?? fallbackPlan.nextActions, + plannerWarnings: [ + ...fallbackPlan.plannerWarnings, + ...((stringArrayValue(parsedPlan.plannerWarnings) ?? []).filter(Boolean)), + ], + createdAt: fallbackPlan.createdAt, + }; +} + +function extractJsonObject(content: string): string { + const firstBraceIndex = content.indexOf("{"); + const lastBraceIndex = content.lastIndexOf("}"); + + if (firstBraceIndex === -1 || lastBraceIndex === -1 || lastBraceIndex <= firstBraceIndex) { + throw new Error("OpenRouter response did not contain a JSON object."); + } + + return content.slice(firstBraceIndex, lastBraceIndex + 1); +} + +function normalizeSchema(value: unknown, fallback: DatasetSchema): DatasetSchema { + if (!isRecord(value)) { + return fallback; + } + + const columns = arrayValue(value.columns) + ?.map(normalizeColumn) + .filter((column): column is DatasetColumnDefinition => Boolean(column)); + + if (!columns?.length) { + return fallback; + } + + const identityColumnName = + stringValue(value.identityColumnName) ?? + columns.find((column) => column.isIdentity)?.name ?? + columns[0]?.name ?? + fallback.identityColumnName; + + return { + datasetName: stringValue(value.datasetName) ?? fallback.datasetName, + identityColumnName, + columns, + }; +} + +function normalizeColumn(value: unknown): DatasetColumnDefinition | null { + if (!isRecord(value)) { + return null; + } + + const name = stringValue(value.name); + const kind = columnKindValue(value.kind); + const description = stringValue(value.description); + + if (!name || !kind || !description) { + return null; + } + + return { + name, + kind, + description, + isRequired: value.isRequired === true, + isIdentity: value.isIdentity === true, + sourceHint: stringValue(value.sourceHint), + }; +} + +function normalizeClarifyingQuestions( + value: unknown, + fallback: ClarifyingQuestion[] +): ClarifyingQuestion[] { + const questions = arrayValue(value); + if (!questions) { + return fallback; + } + + return questions + .map((question, index) => { + if (typeof question === "string") { + return { + id: `openrouter_question_${index + 1}`, + question, + reason: "OpenRouter suggested this scope check.", + }; + } + + if (!isRecord(question)) { + return null; + } + + const questionText = stringValue(question.question); + if (!questionText) { + return null; + } + + return { + id: stringValue(question.id) ?? `openrouter_question_${index + 1}`, + question: questionText, + reason: + stringValue(question.reason) ?? + "OpenRouter suggested this scope check.", + appliesTo: stringValue(question.appliesTo), + }; + }) + .filter((question): question is ClarifyingQuestion => Boolean(question)); +} + +function normalizeHarnessStages( + value: unknown, + fallback: AgentHarnessStage[] +): AgentHarnessStage[] { + const stages = arrayValue(value); + if (!stages) { + return fallback; + } + + const normalizedStages = stages + .map((stage) => { + if (!isRecord(stage)) { + return null; + } + + const id = stringValue(stage.id); + const title = stringValue(stage.title); + const purpose = stringValue(stage.purpose); + const tool = harnessToolValue(stage.tool); + + if (!id || !title || !purpose || !tool) { + return null; + } + + return { + id, + title, + purpose, + tool, + canRunWithoutUser: stage.canRunWithoutUser === true, + }; + }) + .filter((stage): stage is AgentHarnessStage => Boolean(stage)); + + return normalizedStages.length > 0 ? normalizedStages : fallback; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function arrayValue(value: unknown): unknown[] | undefined { + return Array.isArray(value) ? value : undefined; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function stringArrayValue(value: unknown): string[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + + const strings = value.filter((item): item is string => typeof item === "string"); + return strings.length === value.length ? strings : undefined; +} + +function columnKindValue(value: unknown): DatasetColumnDefinition["kind"] | undefined { + if ( + value === "text" || + value === "number" || + value === "boolean" || + value === "date" || + value === "url" || + value === "json" + ) { + return value; + } + + return undefined; +} + +function updateCadenceValue(value: unknown): DatasetUpdateCadence | undefined { + if ( + value === "manual" || + value === "hourly" || + value === "daily" || + value === "weekly" + ) { + return value; + } + + return undefined; +} + +function sourceStrategyValue(value: unknown): DatasetSourceStrategy | undefined { + if ( + value === "search_fetch" || + value === "search_fetch_browser" || + value === "browser_form_fill" + ) { + return value; + } + + return undefined; +} + +function harnessToolValue(value: unknown): AgentHarnessStage["tool"] | undefined { + if ( + value === "user_input" || + value === "tinyfish_search" || + value === "tinyfish_fetch" || + value === "tinyfish_agent" || + value === "validator" || + value === "database" + ) { + return value; + } + + return undefined; +} diff --git a/backend/src/dataset-builder/planner.ts b/backend/src/dataset-builder/planner.ts new file mode 100644 index 0000000..d97f6c7 --- /dev/null +++ b/backend/src/dataset-builder/planner.ts @@ -0,0 +1,300 @@ +import { createHarnessStages } from "./agent-harness.js"; +import type { OpenRouterPlannerClient } from "./openrouter.js"; +import type { + ClarifyingQuestion, + DatasetBuildPlan, + DatasetBuildRequest, + DatasetColumnDefinition, + DatasetSourceStrategy, + DatasetUpdateCadence, +} from "./types.js"; + +interface PlannerOptions { + openRouterClient?: OpenRouterPlannerClient; +} + +export async function createDatasetBuildPlan( + request: DatasetBuildRequest, + options: PlannerOptions = {} +): Promise { + const draftPlan = createDeterministicDatasetBuildPlan(request); + + if (request.planningMode === "openrouter" && options.openRouterClient) { + try { + return await options.openRouterClient.refineDatasetBuildPlan( + request, + draftPlan + ); + } catch (error) { + return { + ...draftPlan, + plannerWarnings: [ + ...draftPlan.plannerWarnings, + error instanceof Error + ? `OpenRouter planner failed: ${error.message}` + : "OpenRouter planner failed with an unknown error.", + ], + }; + } + } + + if (request.planningMode === "openrouter" && !options.openRouterClient) { + return { + ...draftPlan, + plannerWarnings: [ + ...draftPlan.plannerWarnings, + "OpenRouter planner was requested but no API key is loaded; using deterministic fallback.", + ], + }; + } + + return draftPlan; +} + +function createDeterministicDatasetBuildPlan( + request: DatasetBuildRequest +): DatasetBuildPlan { + const normalizedRequest = request.userRequest.trim(); + const lowerRequest = normalizedRequest.toLowerCase(); + const preferredColumns = request.preferredColumns ?? []; + const clarifyingQuestions = inferClarifyingQuestions( + lowerRequest, + request.providedInputs ?? {} + ); + const sourceStrategy = inferSourceStrategy(lowerRequest); + const datasetName = inferDatasetName(normalizedRequest); + const columns = mergeColumns( + inferColumns(lowerRequest), + preferredColumns.map((columnName) => ({ + name: normalizeColumnName(columnName), + kind: "text" as const, + description: `User-requested field: ${columnName}`, + isRequired: false, + })) + ); + + const schema = { + datasetName, + identityColumnName: columns[0]?.name ?? "entity_name", + columns, + }; + + return { + datasetName, + userRequest: normalizedRequest, + updateCadence: request.updateCadence ?? "manual", + schema, + sourceStrategy, + clarifyingQuestions, + harnessStages: createHarnessStages( + sourceStrategy, + clarifyingQuestions.length > 0 + ), + validationRules: [ + "Every populated value must include a source URL.", + "Rows missing the identity column are rejected.", + "Cells that cannot be found are stored as null with status missing.", + "LLM-generated values are not accepted unless backed by fetched or browsed source text.", + "Scheduled updates replace cells for the same identity row instead of appending duplicates.", + ], + replacementPolicy: + "Use the identity column to upsert the current row. Keep run artifacts separately so history can be added later without changing the MVP table contract.", + nextActions: [ + "Confirm or answer any clarifying questions before browser/form work.", + "Run TinyFish Search to discover candidate source URLs.", + "Run TinyFish Fetch on candidate URLs and fill easy fields first.", + "Escalate only hard-to-reach values to TinyFish Agent/browser automation.", + "Validate shape, source URLs, and missing cells before writing rows.", + ], + plannerWarnings: [], + createdAt: new Date().toISOString(), + }; +} + +function inferColumns(lowerRequest: string): DatasetColumnDefinition[] { + const columns: DatasetColumnDefinition[] = [ + { + name: "entity_name", + kind: "text", + description: "Primary row identity, such as company, product, restaurant, or provider name.", + isRequired: true, + isIdentity: true, + }, + { + name: "source_url", + kind: "url", + description: "URL where the row or latest value was found.", + isRequired: true, + }, + { + name: "last_checked_at", + kind: "date", + description: "Timestamp for the latest successful check.", + isRequired: true, + }, + { + name: "confidence_score", + kind: "number", + description: "Simple 0-1 confidence score based on source quality and schema match.", + isRequired: true, + }, + { + name: "extraction_status", + kind: "text", + description: "valid, missing, or needs_review for the row.", + isRequired: true, + }, + ]; + + if (/\b(price|prices|pricing|quote|quotes|cost|costs|rate|rates|premium|premiums)\b/.test(lowerRequest)) { + columns.splice(1, 0, { + name: "current_price", + kind: "number", + description: "Latest price, quote, rate, or premium found for this row.", + isRequired: false, + sourceHint: "Usually requires fetch; may require browser form fill for quotes.", + }); + } + + if (/\b(hiring|jobs|open roles|recruiting)\b/.test(lowerRequest)) { + columns.splice(1, 0, { + name: "is_hiring", + kind: "boolean", + description: "Whether the source currently shows active hiring.", + isRequired: false, + }); + columns.splice(2, 0, { + name: "open_roles_count", + kind: "number", + description: "Count of open roles found on the source.", + isRequired: false, + }); + } + + if (/\b(blog|post|article|newsletter)\b/.test(lowerRequest)) { + columns.splice(1, 0, { + name: "latest_post_title", + kind: "text", + description: "Most recent post title found for this source.", + isRequired: false, + }); + columns.splice(2, 0, { + name: "latest_post_date", + kind: "date", + description: "Publication date for the latest post.", + isRequired: false, + }); + } + + if (/\b(restaurant|serve|coke|pepsi|menu)\b/.test(lowerRequest)) { + columns.splice(1, 0, { + name: "serves_requested_item", + kind: "boolean", + description: "Whether the restaurant appears to serve the requested item or brand.", + isRequired: false, + }); + } + + return columns; +} + +function inferClarifyingQuestions( + lowerRequest: string, + providedInputs: Record +): ClarifyingQuestion[] { + const questions: ClarifyingQuestion[] = []; + const hasLocation = Boolean(providedInputs.location) || /\bin\b\s+[\w\s]+/.test(lowerRequest); + + if (!hasLocation && /\b(restaurant|store|insurance|quote|price)\b/.test(lowerRequest)) { + questions.push({ + id: "location", + question: "Which city, state, or region should this dataset target?", + reason: "Local prices, restaurants, and insurance quotes need geography.", + }); + } + + if (/\b(insurance|quote|quotes|premium|premiums)\b/.test(lowerRequest)) { + if (!providedInputs.age) { + questions.push({ + id: "driver_age", + question: "What driver age should the quote flow use?", + reason: "Insurance forms commonly require age before returning a quote.", + }); + } + if (!providedInputs.vehicle && !hasVehicleDescription(lowerRequest)) { + questions.push({ + id: "vehicle", + question: "What vehicle make, model, and year should the quote flow use?", + reason: "Vehicle details change quote eligibility and price.", + }); + } + } + + if (/\b(competitor|competitors)\b/.test(lowerRequest) && !providedInputs.competitors) { + questions.push({ + id: "competitor_list", + question: "Which competitor names or domains should seed the search?", + reason: "Without seed competitors, the agent may build rows for the wrong market.", + }); + } + + return questions; +} + +function inferSourceStrategy(lowerRequest: string): DatasetSourceStrategy { + if (/\b(insurance|quote|quotes|form|application|checkout|booking)\b/.test(lowerRequest)) { + return "browser_form_fill"; + } + if (/\b(click|login|filter|map|menu|availability)\b/.test(lowerRequest)) { + return "search_fetch_browser"; + } + return "search_fetch"; +} + +function hasVehicleDescription(lowerRequest: string): boolean { + return /\b(19|20)\d{2}\s+[a-z][a-z0-9-]*(?:\s+[a-z][a-z0-9-]*){1,3}\b/.test( + lowerRequest + ); +} + +function inferDatasetName(userRequest: string): string { + const cleanedWords = userRequest + .replace(/[^\w\s-]/g, " ") + .split(/\s+/) + .filter(Boolean) + .slice(0, 8); + + if (cleanedWords.length === 0) { + return "Untitled Dataset"; + } + + return cleanedWords + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(" "); +} + +function normalizeColumnName(columnName: string): string { + return columnName + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); +} + +function mergeColumns( + inferredColumns: DatasetColumnDefinition[], + preferredColumns: DatasetColumnDefinition[] +): DatasetColumnDefinition[] { + const seenColumnNames = new Set(); + const columns: DatasetColumnDefinition[] = []; + + for (const column of [...inferredColumns, ...preferredColumns]) { + if (!column.name || seenColumnNames.has(column.name)) { + continue; + } + seenColumnNames.add(column.name); + columns.push(column); + } + + return columns; +} diff --git a/backend/src/dataset-builder/run-prototype.ts b/backend/src/dataset-builder/run-prototype.ts new file mode 100644 index 0000000..e49cfc6 --- /dev/null +++ b/backend/src/dataset-builder/run-prototype.ts @@ -0,0 +1,46 @@ +import { config } from "dotenv"; + +import { createTinyFishAgentGoal, createTinyFishAgentOutputSchema } from "./agent-harness.js"; +import { createOpenRouterPlannerClient } from "./openrouter.js"; +import { createDatasetBuildPlan } from "./planner.js"; + +config({ path: ".env.local" }); +config({ path: "../.env.local" }); +config(); + +const args = process.argv.slice(2); +const shouldUseOpenRouter = args.includes("--use-openrouter"); +const userRequest = args.filter((arg) => arg !== "--use-openrouter").join(" ").trim(); + +if (!userRequest) { + throw new Error( + "Usage: npm run builder:plan -- \"latest blog posts from my competitors\" [--use-openrouter]" + ); +} + +const plan = await createDatasetBuildPlan( + { + userRequest, + planningMode: shouldUseOpenRouter ? "openrouter" : "deterministic", + }, + { + openRouterClient: shouldUseOpenRouter + ? createOpenRouterPlannerClient({ + apiKey: process.env.OPENROUTER_API_KEY, + model: process.env.OPENROUTER_MODEL || "openai/gpt-4.1-mini", + }) + : undefined, + } +); + +console.log( + JSON.stringify( + { + plan, + tinyFishAgentGoal: createTinyFishAgentGoal(plan), + tinyFishAgentOutputSchema: createTinyFishAgentOutputSchema(plan), + }, + null, + 2 + ) +); diff --git a/backend/src/dataset-builder/tinyfish-cli.ts b/backend/src/dataset-builder/tinyfish-cli.ts new file mode 100644 index 0000000..df72163 --- /dev/null +++ b/backend/src/dataset-builder/tinyfish-cli.ts @@ -0,0 +1,60 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export interface TinyFishSearchOptions { + query: string; + location?: string; + language?: string; +} + +export interface TinyFishAgentRunOptions { + goal: string; + url?: string; + outputSchema?: unknown; + maxSteps?: number; +} + +export async function runTinyFishSearch(options: TinyFishSearchOptions) { + const args = ["search", "query", options.query]; + if (options.location) { + args.push("--location", options.location); + } + if (options.language) { + args.push("--language", options.language); + } + + return runTinyFishJson(args); +} + +export async function runTinyFishFetch(urls: string[]) { + return runTinyFishJson(["fetch", "content", "get", ...urls]); +} + +export async function runTinyFishAgent(options: TinyFishAgentRunOptions) { + const args = ["agent", "run", "--sync"]; + + if (options.url) { + args.push("--url", options.url); + } + if (options.maxSteps) { + args.push("--max-steps", String(options.maxSteps)); + } + if (options.outputSchema) { + args.push("--output-schema", JSON.stringify(options.outputSchema)); + } + + args.push(options.goal); + + return runTinyFishJson(args); +} + +async function runTinyFishJson(args: string[]) { + const { stdout } = await execFileAsync("tinyfish", args, { + env: process.env, + maxBuffer: 1024 * 1024 * 10, + }); + + return JSON.parse(stdout) as unknown; +} diff --git a/backend/src/dataset-builder/types.ts b/backend/src/dataset-builder/types.ts new file mode 100644 index 0000000..255c3cc --- /dev/null +++ b/backend/src/dataset-builder/types.ts @@ -0,0 +1,91 @@ +export type DatasetColumnKind = + | "text" + | "number" + | "boolean" + | "date" + | "url" + | "json"; + +export type DatasetUpdateCadence = "manual" | "hourly" | "daily" | "weekly"; + +export type DatasetSourceStrategy = + | "search_fetch" + | "search_fetch_browser" + | "browser_form_fill"; + +export type DatasetPlanningMode = "deterministic" | "openrouter"; + +export interface DatasetColumnDefinition { + name: string; + kind: DatasetColumnKind; + description: string; + isRequired: boolean; + isIdentity?: boolean; + sourceHint?: string; +} + +export interface DatasetSchema { + datasetName: string; + identityColumnName: string; + columns: DatasetColumnDefinition[]; +} + +export interface ClarifyingQuestion { + id: string; + question: string; + reason: string; + appliesTo?: string; +} + +export interface DatasetBuildRequest { + userRequest: string; + updateCadence?: DatasetUpdateCadence; + providedInputs?: Record; + preferredColumns?: string[]; + planningMode?: DatasetPlanningMode; +} + +export interface AgentHarnessStage { + id: string; + title: string; + purpose: string; + tool: "user_input" | "tinyfish_search" | "tinyfish_fetch" | "tinyfish_agent" | "validator" | "database"; + canRunWithoutUser: boolean; +} + +export interface DatasetBuildPlan { + datasetName: string; + userRequest: string; + updateCadence: DatasetUpdateCadence; + schema: DatasetSchema; + sourceStrategy: DatasetSourceStrategy; + clarifyingQuestions: ClarifyingQuestion[]; + harnessStages: AgentHarnessStage[]; + validationRules: string[]; + replacementPolicy: string; + nextActions: string[]; + plannerWarnings: string[]; + createdAt: string; +} + +export interface DatasetRunCell { + columnName: string; + value: string | number | boolean | null; + sourceUrl?: string; + confidenceScore: number; + validationStatus: "valid" | "missing" | "needs_review"; +} + +export interface DatasetRunRow { + identityValue: string; + cells: DatasetRunCell[]; +} + +export interface DatasetRunArtifact { + planId?: string; + rows: DatasetRunRow[]; + sourceUrls: string[]; + missingInputs: ClarifyingQuestion[]; + validationIssues: string[]; + completedAt?: string; +} diff --git a/backend/src/env.ts b/backend/src/env.ts index 761c6e4..5bfac58 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -1,4 +1,8 @@ -import "dotenv/config"; +import { config } from "dotenv"; + +config({ path: ".env.local" }); +config({ path: "../.env.local" }); +config(); function required(name: string): string { const value = process.env[name]; @@ -8,10 +12,21 @@ function required(name: string): string { return value; } +function requiredPort(name: string): number { + const value = Number(required(name)); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + export const env = { BETTER_AUTH_SECRET: required("BETTER_AUTH_SECRET"), BETTER_AUTH_URL: required("BETTER_AUTH_URL"), - CLIENT_ORIGIN: process.env.CLIENT_ORIGIN || "http://localhost:3500", + CLIENT_ORIGIN: required("CLIENT_ORIGIN"), DATABASE_URL: required("DATABASE_URL"), - PORT: Number(process.env.PORT || "3501"), + OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY, + OPENROUTER_MODEL: process.env.OPENROUTER_MODEL || "openai/gpt-4.1-mini", + PORT: requiredPort("PORT"), + TINYFISH_API_KEY: process.env.TINYFISH_API_KEY, }; diff --git a/backend/src/index.ts b/backend/src/index.ts index 9ede16d..a2ef7c1 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -4,6 +4,7 @@ import { fromNodeHeaders } from "better-auth/node"; import { auth } from "./auth.js"; import { env } from "./env.js"; +import { registerDatasetBuilderRoutes } from "./routes/dataset-builder.js"; const fastify = Fastify({ logger: true }); @@ -59,6 +60,8 @@ fastify.get("/api/me", async (request, reply) => { return reply.send(session); }); +await registerDatasetBuilderRoutes(fastify); + try { await fastify.listen({ port: env.PORT, host: "0.0.0.0" }); } catch (err) { diff --git a/backend/src/routes/dataset-builder.ts b/backend/src/routes/dataset-builder.ts new file mode 100644 index 0000000..b2540a8 --- /dev/null +++ b/backend/src/routes/dataset-builder.ts @@ -0,0 +1,151 @@ +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import { randomUUID } from "node:crypto"; +import { fromNodeHeaders } from "better-auth/node"; + +import { auth } from "../auth.js"; +import { env } from "../env.js"; +import { createTinyFishAgentGoal, createTinyFishAgentOutputSchema } from "../dataset-builder/agent-harness.js"; +import { createOpenRouterPlannerClient } from "../dataset-builder/openrouter.js"; +import { createDatasetBuildPlan } from "../dataset-builder/planner.js"; +import type { + DatasetBuildRequest, + DatasetPlanningMode, + DatasetUpdateCadence, +} from "../dataset-builder/types.js"; + +export async function registerDatasetBuilderRoutes(fastify: FastifyInstance) { + fastify.post("/api/dataset-builder/plan", async (request, reply) => { + const userId = await requireAuthenticatedUserId(request, reply); + if (!userId) { + return; + } + + const parsedRequest = parseDatasetBuildRequestBody(request.body); + if (!parsedRequest.ok) { + return reply.status(400).send({ + error: "Invalid dataset build request", + details: parsedRequest.error, + }); + } + + const plan = await createDatasetBuildPlan(parsedRequest.value, { + openRouterClient: createOpenRouterPlannerClient({ + apiKey: env.OPENROUTER_API_KEY, + model: env.OPENROUTER_MODEL, + }), + }); + + return reply.send({ + planId: randomUUID(), + ownerUserId: userId, + plan, + tinyFishAgentGoal: createTinyFishAgentGoal(plan), + tinyFishAgentOutputSchema: createTinyFishAgentOutputSchema(plan), + }); + }); +} + +async function requireAuthenticatedUserId( + request: FastifyRequest, + reply: FastifyReply +): Promise { + const session = await auth.api.getSession({ + headers: fromNodeHeaders(request.headers), + }); + + if (!session) { + reply.status(401).send({ error: "Unauthorized" }); + return null; + } + + return session.user.id; +} + +function parseDatasetBuildRequestBody( + body: unknown +): { ok: true; value: DatasetBuildRequest } | { ok: false; error: string } { + if (!isRecord(body)) { + return { ok: false, error: "Body must be a JSON object." }; + } + + const userRequest = stringValue(body.userRequest) ?? stringValue(body.prompt); + if (!userRequest?.trim()) { + return { ok: false, error: "`userRequest` is required." }; + } + + const updateCadence = parseUpdateCadence(body.updateCadence); + if (body.updateCadence && !updateCadence) { + return { + ok: false, + error: "`updateCadence` must be manual, hourly, daily, or weekly.", + }; + } + + const planningMode = parsePlanningMode(body.planningMode); + if (body.planningMode && !planningMode) { + return { + ok: false, + error: "`planningMode` must be deterministic or openrouter.", + }; + } + + return { + ok: true, + value: { + userRequest, + updateCadence, + planningMode: planningMode ?? "openrouter", + providedInputs: parseStringRecord(body.providedInputs), + preferredColumns: parseStringArray(body.preferredColumns), + }, + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function parseStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + + return value.filter((item): item is string => typeof item === "string"); +} + +function parseStringRecord(value: unknown): Record | undefined { + if (!isRecord(value)) { + return undefined; + } + + return Object.fromEntries( + Object.entries(value).filter( + (entry): entry is [string, string] => typeof entry[1] === "string" + ) + ); +} + +function parseUpdateCadence(value: unknown): DatasetUpdateCadence | undefined { + if ( + value === "manual" || + value === "hourly" || + value === "daily" || + value === "weekly" + ) { + return value; + } + + return undefined; +} + +function parsePlanningMode(value: unknown): DatasetPlanningMode | undefined { + if (value === "deterministic" || value === "openrouter") { + return value; + } + + return undefined; +} diff --git a/backend/src/schema.ts b/backend/src/schema.ts index e45c2c0..e35f205 100644 --- a/backend/src/schema.ts +++ b/backend/src/schema.ts @@ -1,4 +1,19 @@ -import { pgTable, text, boolean, timestamp } from "drizzle-orm/pg-core"; +import { + boolean, + index, + integer, + jsonb, + pgTable, + text, + timestamp, +} from "drizzle-orm/pg-core"; + +import type { + DatasetBuildPlan, + DatasetRunArtifact, + DatasetSchema, + DatasetUpdateCadence, +} from "./dataset-builder/types.js"; export const user = pgTable("user", { id: text("id").primaryKey(), @@ -49,3 +64,55 @@ export const verification = pgTable("verification", { createdAt: timestamp("created_at"), updatedAt: timestamp("updated_at"), }); + +export const dataset = pgTable( + "dataset", + { + id: text("id").primaryKey(), + ownerUserId: text("owner_user_id") + .notNull() + .references(() => user.id, { onDelete: "cascade" }), + name: text("name").notNull(), + userRequest: text("user_request").notNull(), + updateCadence: text("update_cadence") + .$type() + .notNull(), + status: text("status") + .$type<"draft" | "needs_input" | "ready" | "running" | "failed">() + .notNull() + .default("draft"), + schema: jsonb("schema").$type().notNull(), + buildPlan: jsonb("build_plan").$type().notNull(), + createdAt: timestamp("created_at").notNull(), + updatedAt: timestamp("updated_at").notNull(), + }, + (table) => [ + index("dataset_owner_user_id_idx").on(table.ownerUserId), + index("dataset_status_idx").on(table.status), + ] +); + +export const datasetRun = pgTable( + "dataset_run", + { + id: text("id").primaryKey(), + datasetId: text("dataset_id") + .notNull() + .references(() => dataset.id, { onDelete: "cascade" }), + status: text("status") + .$type<"queued" | "running" | "needs_input" | "succeeded" | "failed">() + .notNull() + .default("queued"), + attemptNumber: integer("attempt_number").notNull().default(1), + artifact: jsonb("artifact").$type(), + errorMessage: text("error_message"), + startedAt: timestamp("started_at"), + completedAt: timestamp("completed_at"), + createdAt: timestamp("created_at").notNull(), + updatedAt: timestamp("updated_at").notNull(), + }, + (table) => [ + index("dataset_run_dataset_id_idx").on(table.datasetId), + index("dataset_run_status_idx").on(table.status), + ] +); diff --git a/docs/dataset-builder-agent-harness.md b/docs/dataset-builder-agent-harness.md new file mode 100644 index 0000000..e4069f4 --- /dev/null +++ b/docs/dataset-builder-agent-harness.md @@ -0,0 +1,72 @@ +# Dataset Builder Agent Harness + +## MVP Decision + +Start with a backend-owned planning harness before table creation gets clever. +The first shippable slice is: + +1. Turn a natural-language dataset request into a fixed schema. +2. Ask missing-input questions before browser/form automation when possible. +3. Prefer TinyFish Search + Fetch first. +4. Escalate only hard pages or form flows to TinyFish Agent/browser automation. +5. Validate cells against schema and source URL requirements. +6. Replace values for the same identity row on refresh instead of appending duplicates. + +History, user trust flags, public dataset resale, column editing, and Convex sync stay future scope. + +## Current Scaffold + +- `backend/src/dataset-builder/types.ts` defines dataset schema, plan, clarifying questions, harness stages, and run artifacts. +- `backend/src/dataset-builder/planner.ts` creates a deterministic draft plan from a user request. +- `backend/src/dataset-builder/openrouter.ts` optionally refines the draft through OpenRouter chat completions. +- `backend/src/dataset-builder/agent-harness.ts` converts a plan into TinyFish agent goals and output schemas. +- `backend/src/dataset-builder/tinyfish-cli.ts` is a local prototype adapter for TinyFish Search, Fetch, and Agent CLI runs. +- `backend/src/routes/dataset-builder.ts` exposes `POST /api/dataset-builder/plan` behind Better Auth. +- `backend/src/schema.ts` now has `dataset` and `dataset_run` metadata tables for plan/run storage. + +## Prototype Command + +From `backend/`: + +```bash +npm run builder:plan -- "restaurants in Menlo Park that serve Coca-Cola" +``` + +Use OpenRouter when a key is loaded: + +```bash +npm run builder:plan -- "car insurance quotes for a 2021 Honda Civic in Menlo Park" --use-openrouter +``` + +The command prints the plan, generated TinyFish agent goal, and output schema. It never prints API keys. + +## API Contract + +`POST /api/dataset-builder/plan` + +```json +{ + "userRequest": "latest blog posts from my competitors", + "updateCadence": "daily", + "planningMode": "openrouter", + "providedInputs": { + "competitors": "exa.ai, perplexity.ai" + }, + "preferredColumns": ["latest post URL"] +} +``` + +Response includes: + +- `planId` +- `plan` +- `tinyFishAgentGoal` +- `tinyFishAgentOutputSchema` + +## Next Tickets + +1. Persist generated plans in `dataset`. +2. Add `POST /api/datasets/:id/runs` to run the harness and write `dataset_run` artifacts. +3. Decide if TinyFish execution should use direct HTTP APIs or CLI only for local experiments. +4. Add a DB-backed queue/lease before cron refresh jobs. +5. Add frontend create-dataset flow once Divya's table UI is ready.