diff --git a/apps/cli/release-notes/next.md b/apps/cli/release-notes/next.md index bdd342e1e..6bf5dc86a 100644 --- a/apps/cli/release-notes/next.md +++ b/apps/cli/release-notes/next.md @@ -78,6 +78,7 @@ Tool dispatch, plugins, storage, schema, and transport are now fully instrumente - Upgrade: preserve legacy OAuth connection backfills after the `connection.kind` column is removed. - OpenAPI: refreshing or editing sources with legacy inline secret/OAuth config now materializes the new source binding rows instead of dropping credentials. - Keychain: skip provider registration when the OS backend is unreachable (no more startup failure when running headless on Linux without a keyring). +- Local database: fail early with guidance when an older Executor build opens a data directory migrated by a newer build, instead of surfacing a low-level SQLite schema error. - Local server: return 404 for missing static assets instead of serving HTML. - Tests: Windows compatibility across the suite. diff --git a/apps/local/src/server/executor-schema-compat.test.ts b/apps/local/src/server/executor-schema-compat.test.ts new file mode 100644 index 000000000..e3dcd9696 --- /dev/null +++ b/apps/local/src/server/executor-schema-compat.test.ts @@ -0,0 +1,156 @@ +import { Database } from "bun:sqlite"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { drizzle } from "drizzle-orm/bun-sqlite"; +import { migrate } from "drizzle-orm/bun-sqlite/migrator"; + +import { + LocalDatabaseMigrationHistoryMismatch, + LocalDatabaseSchemaTooNew, + checkDrizzleMigrationCompatibility, + readAppliedDrizzleMigrationHashes, + readBundledDrizzleMigrationHashes, +} from "./executor"; + +const MIGRATIONS_FOLDER = join(import.meta.dirname, "../../drizzle"); + +const workDirs: string[] = []; +const openDbs: Database[] = []; + +const tempDb = (): { db: Database; path: string; dataDir: string } => { + const dir = mkdtempSync(join(tmpdir(), "executor-schema-compat-")); + workDirs.push(dir); + const path = join(dir, "data.db"); + const db = new Database(path); + openDbs.push(db); + return { db, path, dataDir: dir }; +}; + +const createMigrationTable = (db: Database): void => { + db.exec(` + CREATE TABLE __drizzle_migrations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT NOT NULL, + created_at NUMERIC + ) + `); +}; + +const insertMigrationHashes = (db: Database, hashes: ReadonlyArray): void => { + const stmt = db.prepare("INSERT INTO __drizzle_migrations (hash, created_at) VALUES (?, ?)"); + for (const [index, hash] of hashes.entries()) { + stmt.run(hash, index + 1); + } +}; + +afterEach(() => { + for (const db of openDbs.splice(0)) { + db.close(); + } + for (const dir of workDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("Drizzle migration compatibility preflight", () => { + it.effect("allows a fresh DB without __drizzle_migrations", () => + Effect.gen(function* () { + const { db, path, dataDir } = tempDb(); + + yield* checkDrizzleMigrationCompatibility({ + sqlite: db, + dbPath: path, + dataDir, + migrationsFolder: MIGRATIONS_FOLDER, + }); + }), + ); + + it.effect("allows an existing but empty __drizzle_migrations table", () => + Effect.gen(function* () { + const { db, path, dataDir } = tempDb(); + createMigrationTable(db); + + yield* checkDrizzleMigrationCompatibility({ + sqlite: db, + dbPath: path, + dataDir, + migrationsFolder: MIGRATIONS_FOLDER, + }); + }), + ); + + it("computes bundled hashes that exactly match hashes written by Drizzle", () => { + const { db } = tempDb(); + migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER }); + + expect(readAppliedDrizzleMigrationHashes(db)).toEqual( + readBundledDrizzleMigrationHashes(MIGRATIONS_FOLDER), + ); + }); + + it.effect("fails with LocalDatabaseSchemaTooNew when the DB has more migrations", () => + Effect.gen(function* () { + const { db, path, dataDir } = tempDb(); + const bundled = readBundledDrizzleMigrationHashes(MIGRATIONS_FOLDER); + createMigrationTable(db); + insertMigrationHashes(db, [...bundled, "future-migration-hash"]); + + const error = yield* checkDrizzleMigrationCompatibility({ + sqlite: db, + dbPath: path, + dataDir, + migrationsFolder: MIGRATIONS_FOLDER, + }).pipe(Effect.flip); + + expect(error).toBeInstanceOf(LocalDatabaseSchemaTooNew); + expect(error).toMatchObject({ + message: expect.stringContaining("This Executor binary is older than the schema"), + }); + expect(error).toMatchObject({ + message: expect.stringContaining("Use a newer Executor binary"), + }); + }), + ); + + it.effect("fails with LocalDatabaseMigrationHistoryMismatch when hashes diverge", () => + Effect.gen(function* () { + const { db, path, dataDir } = tempDb(); + const bundled = readBundledDrizzleMigrationHashes(MIGRATIONS_FOLDER); + createMigrationTable(db); + insertMigrationHashes(db, ["different-migration-hash", ...bundled.slice(1)]); + + const error = yield* checkDrizzleMigrationCompatibility({ + sqlite: db, + dbPath: path, + dataDir, + migrationsFolder: MIGRATIONS_FOLDER, + }).pipe(Effect.flip); + + expect(error).toBeInstanceOf(LocalDatabaseMigrationHistoryMismatch); + expect(error).toMatchObject({ + message: expect.stringContaining("does not match this Executor build"), + }); + expect(error).toMatchObject({ message: expect.stringContaining("restore a backup") }); + }), + ); + + it.effect("allows an older DB whose migration history is a bundled prefix", () => + Effect.gen(function* () { + const { db, path, dataDir } = tempDb(); + const bundled = readBundledDrizzleMigrationHashes(MIGRATIONS_FOLDER); + createMigrationTable(db); + insertMigrationHashes(db, bundled.slice(0, 1)); + + yield* checkDrizzleMigrationCompatibility({ + sqlite: db, + dbPath: path, + dataDir, + migrationsFolder: MIGRATIONS_FOLDER, + }); + }), + ); +}); diff --git a/apps/local/src/server/executor.ts b/apps/local/src/server/executor.ts index 617bdf9f9..466b7550d 100644 --- a/apps/local/src/server/executor.ts +++ b/apps/local/src/server/executor.ts @@ -1,7 +1,7 @@ import { Database } from "bun:sqlite"; import { drizzle } from "drizzle-orm/bun-sqlite"; import { migrate } from "drizzle-orm/bun-sqlite/migrator"; -import { Context, Data, Effect, Layer, ManagedRuntime } from "effect"; +import { Context, Data, Effect, Layer, ManagedRuntime, Schema } from "effect"; import { createHash } from "node:crypto"; import * as fs from "node:fs"; import { homedir, tmpdir } from "node:os"; @@ -45,6 +45,7 @@ const MIGRATIONS_FOLDER = resolveMigrationsFolder(); interface ResolvedDb { readonly path: string; + readonly dataDir: string; readonly legacySecrets: readonly LegacySecret[]; } @@ -67,7 +68,7 @@ const resolveDbPath = (): ResolvedDb => { : "."), ); } - return { path: dbPath, legacySecrets }; + return { path: dbPath, dataDir, legacySecrets }; }; // Hash suffix disambiguates same-basename folders so two projects with @@ -97,6 +98,23 @@ class LocalExecutorTag extends Context.Service {} + +export class LocalDatabaseMigrationHistoryMismatch extends Data.TaggedError( + "LocalDatabaseMigrationHistoryMismatch", +)<{ + readonly message: string; + readonly dbPath: string; + readonly migrationIndex: number; + readonly appliedHash: string | undefined; + readonly knownHash: string | undefined; +}> {} + class LocalExecutorDisposeError extends Data.TaggedError("LocalExecutorDisposeError")<{ readonly operation: "createHandle" | "disposeExecutor" | "disposeRuntime"; readonly cause: unknown; @@ -127,8 +145,108 @@ const handleOrNull = (promise: ReturnType) => ), ); +export const drizzleMigrationsTableExists = (sqlite: Database): boolean => { + const row = sqlite + .query<{ name: string }, [string]>( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", + ) + .get("__drizzle_migrations"); + + return row != null; +}; + +export const readAppliedDrizzleMigrationHashes = (sqlite: Database): ReadonlyArray => { + if (!drizzleMigrationsTableExists(sqlite)) return []; + + // Drizzle inserts one row per applied migration. `id` is the stable + // application order; `created_at` comes from migration metadata and can tie. + return sqlite + .query<{ hash: string }, []>("SELECT hash FROM __drizzle_migrations ORDER BY id ASC") + .all() + .map((row) => row.hash); +}; + +const DrizzleJournal = Schema.Struct({ + entries: Schema.Array( + Schema.Struct({ + idx: Schema.Number, + tag: Schema.String, + }), + ), +}); + +const decodeDrizzleJournal = Schema.decodeUnknownSync(Schema.fromJsonString(DrizzleJournal)); + +export const readBundledDrizzleMigrationHashes = ( + migrationsFolder: string, +): ReadonlyArray => { + // Keep this in sync with drizzle-orm/src/migrator.ts: Drizzle hashes the raw + // migration file contents before splitting on statement breakpoints. + const journal = decodeDrizzleJournal( + fs.readFileSync(join(migrationsFolder, "meta", "_journal.json")).toString(), + ); + + return [...journal.entries] + .sort((left, right) => left.idx - right.idx) + .map((entry) => { + const query = fs.readFileSync(join(migrationsFolder, `${entry.tag}.sql`)).toString(); + return createHash("sha256").update(query).digest("hex"); + }); +}; + +const schemaTooNewMessage = (dataDir: string): string => + [ + `This Executor binary is older than the schema in ${dataDir}.`, + "The database was likely opened by a newer Executor build.", + "Use a newer Executor binary or set EXECUTOR_DATA_DIR to a different data directory.", + ].join("\n"); + +const migrationHistoryMismatchMessage = (dataDir: string): string => + [ + `The migration history in ${dataDir} does not match this Executor build.`, + "The database may have been created by a different development branch, manually modified, or corrupted.", + "Use the matching Executor build, set EXECUTOR_DATA_DIR to a different data directory, or restore a backup.", + ].join("\n"); + +export const checkDrizzleMigrationCompatibility = (input: { + readonly sqlite: Database; + readonly dbPath: string; + readonly dataDir: string; + readonly migrationsFolder: string; +}): Effect.Effect => + Effect.gen(function* () { + // Before running migrations, ensure the DB history is a prefix of the + // migrations bundled with this binary. This catches newer or divergent schemas + // before startup reaches arbitrary schema-dependent queries. + if (!drizzleMigrationsTableExists(input.sqlite)) return; + + const applied = readAppliedDrizzleMigrationHashes(input.sqlite); + const bundled = readBundledDrizzleMigrationHashes(input.migrationsFolder); + + if (applied.length > bundled.length) { + return yield* new LocalDatabaseSchemaTooNew({ + message: schemaTooNewMessage(input.dataDir), + dbPath: input.dbPath, + appliedMigrationCount: applied.length, + knownMigrationCount: bundled.length, + }); + } + + for (let index = 0; index < applied.length; index += 1) { + if (applied[index] !== bundled[index]) { + return yield* new LocalDatabaseMigrationHistoryMismatch({ + message: migrationHistoryMismatchMessage(input.dataDir), + dbPath: input.dbPath, + migrationIndex: index, + appliedHash: applied[index], + knownHash: bundled[index], + }); + } + } + }); + const createLocalExecutorLayer = () => { - const { path: dbPath, legacySecrets } = resolveDbPath(); + const { path: dbPath, dataDir, legacySecrets } = resolveDbPath(); return Layer.effect(LocalExecutorTag)( Effect.gen(function* () { @@ -136,6 +254,12 @@ const createLocalExecutorLayer = () => { Effect.sync(() => new Database(dbPath)), (conn) => Effect.sync(() => conn.close()), ); + yield* checkDrizzleMigrationCompatibility({ + sqlite, + dbPath, + dataDir, + migrationsFolder: MIGRATIONS_FOLDER, + }); sqlite.exec("PRAGMA journal_mode = WAL"); const db = drizzle(sqlite, { schema: executorSchema });