From c9f6783487468b089529d7a003e1376577a0cd10 Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Tue, 16 Jun 2026 10:32:17 -0600 Subject: [PATCH 01/10] build(embedded): use local NIO/log clones and gate NIOPosix off WASI NIOPosix carries a resource bundle whose generated accessor imports Foundation, unavailable on the embedded WASI target; gate it to non-WASI and rely on NIOAsyncRuntime there. Co-Authored-By: Claude Opus 4.8 (1M context) --- Package.swift | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Package.swift b/Package.swift index e448c21..7dc4fba 100644 --- a/Package.swift +++ b/Package.swift @@ -19,10 +19,9 @@ let package = Package( .library(name: "SQLiteNIO", targets: ["SQLiteNIO"]), ], dependencies: [ - // TODO: SM: Update swift-nio version once NIOAsyncRuntime is available from swift-nio - // .package(url: "https://github.com/apple/swift-nio.git", from: "2.89.0"), - .package(url: "https://github.com/PassiveLogic/swift-nio.git", branch: "feat/khasmPAL-2026"), - .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), + // Local clones for the embedded-wasm port (see /Users/scottm/git/c34/EMBEDDED_WASM_NOTES.md) + .package(path: "../swift-nio"), + .package(path: "../swift-log"), ], targets: [ .plugin( @@ -47,7 +46,9 @@ let package = Package( .product(name: "Logging", package: "swift-log"), .product(name: "NIOCore", package: "swift-nio"), .product(name: "NIOAsyncRuntime", package: "swift-nio", condition: .when(platforms: wasiPlatform)), - .product(name: "NIOPosix", package: "swift-nio"), + // NIOPosix carries a resource bundle whose generated accessor imports Foundation, + // which is unavailable on the embedded WASI target; use NIOAsyncRuntime there instead. + .product(name: "NIOPosix", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms)), .product(name: "NIOFoundationCompat", package: "swift-nio"), ], swiftSettings: swiftSettings From bfb68ca88e6300bd6791045298ed5fd6adf099f5 Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Tue, 16 Jun 2026 11:17:43 -0600 Subject: [PATCH 02/10] feat(embedded): add SwiftNIO/EmbeddedWASI SPM traits (Package@swift-6.1) EmbeddedWASI drops all swift-nio products (NIOCore/NIOPosix/NIOAsyncRuntime/NIOFoundationCompat), leaving CSQLite + swift-log for a NIO-free Swift-Concurrency SQLite driver. SwiftNIO (default) is unchanged. Non-breaking for older toolchains. Co-Authored-By: Claude Opus 4.8 (1M context) --- Package@swift-6.1.swift | 118 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 Package@swift-6.1.swift diff --git a/Package@swift-6.1.swift b/Package@swift-6.1.swift new file mode 100644 index 0000000..fbae1fb --- /dev/null +++ b/Package@swift-6.1.swift @@ -0,0 +1,118 @@ +// swift-tools-version:6.1 +import PackageDescription + +// Trait-aware manifest (SE-0450). Modern toolchains select this over Package.swift. +// - `SwiftNIO` (default): existing SwiftNIO backend — unchanged behavior (incl. the regular-WASI +// NIOAsyncRuntime substitution). +// - `EmbeddedWASI`: NIO-free, Swift-Concurrency SQLite driver over CSQLite for Embedded Swift / +// WASI. Enable it *instead of* the defaults; turns `SwiftNIO` off and drops swift-nio. +let allPlatforms: [Platform] = [.macOS, .macCatalyst, .iOS, .tvOS, .watchOS, .visionOS, .driverKit, .linux, .windows, .android, .wasi, .openbsd] +let nonWASIPlatforms: [Platform] = allPlatforms.filter { $0 != .wasi } +let wasiPlatform: [Platform] = [.wasi] + +let package = Package( + name: "sqlite-nio", + platforms: [ + .macOS(.v10_15), + .iOS(.v13), + .watchOS(.v6), + .tvOS(.v13), + ], + products: [ + .library(name: "SQLiteNIO", targets: ["SQLiteNIO"]), + ], + traits: [ + .trait(name: "SwiftNIO", description: "Default backend built on SwiftNIO (EventLoopFuture)."), + .trait( + name: "EmbeddedWASI", + description: "NIO-free, Swift-Concurrency SQLite driver for Embedded Swift / WASI; drops swift-nio." + ), + .default(enabledTraits: ["SwiftNIO"]), + ], + dependencies: [ + .package(path: "../swift-nio"), + .package(path: "../swift-log"), + ], + targets: [ + .plugin( + name: "VendorSQLite", + capability: .command( + intent: .custom(verb: "vendor-sqlite", description: "Vendor SQLite"), + permissions: [ + .allowNetworkConnections(scope: .all(ports: [443]), reason: "Retrieve the latest build of SQLite"), + .writeToPackageDirectory(reason: "Update the vendored SQLite files"), + ] + ), + exclude: ["001-warnings-and-data-race.patch"] + ), + .target( + name: "CSQLite", + cSettings: sqliteCSettings + ), + .target( + name: "SQLiteNIO", + dependencies: [ + .target(name: "CSQLite"), + .product(name: "Logging", package: "swift-log"), + // SwiftNIO backend products — only when the SwiftNIO trait is enabled. + .product(name: "NIOCore", package: "swift-nio", condition: .when(traits: ["SwiftNIO"])), + .product(name: "NIOAsyncRuntime", package: "swift-nio", condition: .when(platforms: wasiPlatform, traits: ["SwiftNIO"])), + .product(name: "NIOPosix", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms, traits: ["SwiftNIO"])), + .product(name: "NIOFoundationCompat", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms, traits: ["SwiftNIO"])), + ], + swiftSettings: swiftSettings + ), + .testTarget( + name: "SQLiteNIOTests", + dependencies: [ + .target(name: "SQLiteNIO"), + ], + swiftSettings: swiftSettings + ), + ] +) + +var swiftSettings: [SwiftSetting] { [ + .enableUpcomingFeature("ExistentialAny"), + .enableUpcomingFeature("ConciseMagicFile"), + .enableUpcomingFeature("ForwardTrailingClosures"), + .enableUpcomingFeature("DisableOutwardActorInference"), + .enableExperimentalFeature("StrictConcurrency=complete"), +] } + +var sqliteCSettings: [CSetting] { [ + // Derived from sqlite3 version 3.43.0 + .define("SQLITE_DEFAULT_MEMSTATUS", to: "0"), + .define("SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS"), + .define("SQLITE_DQS", to: "0"), + .define("SQLITE_ENABLE_API_ARMOR", .when(configuration: .debug)), + .define("SQLITE_ENABLE_COLUMN_METADATA"), + .define("SQLITE_ENABLE_DBSTAT_VTAB"), + .define("SQLITE_ENABLE_FTS3"), + .define("SQLITE_ENABLE_FTS3_PARENTHESIS"), + .define("SQLITE_ENABLE_FTS3_TOKENIZER"), + .define("SQLITE_ENABLE_FTS4"), + .define("SQLITE_ENABLE_FTS5"), + .define("SQLITE_ENABLE_NULL_TRIM"), + .define("SQLITE_ENABLE_RTREE"), + .define("SQLITE_ENABLE_SESSION"), + .define("SQLITE_ENABLE_STMTVTAB"), + .define("SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION"), + .define("SQLITE_ENABLE_UNLOCK_NOTIFY"), + .define("SQLITE_MAX_VARIABLE_NUMBER", to: "250000"), + .define("SQLITE_LIKE_DOESNT_MATCH_BLOBS"), + .define("SQLITE_OMIT_COMPLETE"), + .define("SQLITE_OMIT_DEPRECATED"), + .define("SQLITE_OMIT_DESERIALIZE"), + .define("SQLITE_OMIT_GET_TABLE"), + .define("SQLITE_OMIT_LOAD_EXTENSION"), + .define("SQLITE_OMIT_PROGRESS_CALLBACK"), + .define("SQLITE_OMIT_SHARED_CACHE"), + .define("SQLITE_OMIT_TCL_VARIABLE"), + .define("SQLITE_OMIT_TRACE"), + .define("SQLITE_SECURE_DELETE"), + .define("SQLITE_THREADSAFE", to: "1", .when(platforms: nonWASIPlatforms)), + .define("SQLITE_THREADSAFE", to: "0", .when(platforms: wasiPlatform)), + .define("SQLITE_UNTESTABLE"), + .define("SQLITE_USE_URI"), +] } From 5e40d7d101c8ec509a441db63a01ba6130c7e1a0 Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Tue, 16 Jun 2026 11:59:41 -0600 Subject: [PATCH 03/10] build(embedded): elide swift-nio on WASI via platform condition (drop trait) Gate NIOCore/NIOPosix/NIOFoundationCompat on .when(platforms: nonWASIPlatforms); remove NIOAsyncRuntime. On WASI only CSQLite + swift-log remain, for a NIO-free Swift-Concurrency driver gated with #if os(WASI). Co-Authored-By: Claude Opus 4.8 (1M context) --- Package.swift | 11 ++-- Package@swift-6.1.swift | 118 ---------------------------------------- 2 files changed, 6 insertions(+), 123 deletions(-) delete mode 100644 Package@swift-6.1.swift diff --git a/Package.swift b/Package.swift index 7dc4fba..5584766 100644 --- a/Package.swift +++ b/Package.swift @@ -44,12 +44,13 @@ let package = Package( dependencies: [ .target(name: "CSQLite"), .product(name: "Logging", package: "swift-log"), - .product(name: "NIOCore", package: "swift-nio"), - .product(name: "NIOAsyncRuntime", package: "swift-nio", condition: .when(platforms: wasiPlatform)), - // NIOPosix carries a resource bundle whose generated accessor imports Foundation, - // which is unavailable on the embedded WASI target; use NIOAsyncRuntime there instead. + // The SwiftNIO stack is elided on WASI (normal + Embedded); .when(platforms:) is + // target-evaluated, so these aren't built when cross-compiling to wasm32-unknown-wasip1. + // The WASI path is a NIO-free, Swift-Concurrency driver over CSQLite, gated in source + // with `#if os(WASI)`. + .product(name: "NIOCore", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms)), .product(name: "NIOPosix", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms)), - .product(name: "NIOFoundationCompat", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms)), ], swiftSettings: swiftSettings ), diff --git a/Package@swift-6.1.swift b/Package@swift-6.1.swift deleted file mode 100644 index fbae1fb..0000000 --- a/Package@swift-6.1.swift +++ /dev/null @@ -1,118 +0,0 @@ -// swift-tools-version:6.1 -import PackageDescription - -// Trait-aware manifest (SE-0450). Modern toolchains select this over Package.swift. -// - `SwiftNIO` (default): existing SwiftNIO backend — unchanged behavior (incl. the regular-WASI -// NIOAsyncRuntime substitution). -// - `EmbeddedWASI`: NIO-free, Swift-Concurrency SQLite driver over CSQLite for Embedded Swift / -// WASI. Enable it *instead of* the defaults; turns `SwiftNIO` off and drops swift-nio. -let allPlatforms: [Platform] = [.macOS, .macCatalyst, .iOS, .tvOS, .watchOS, .visionOS, .driverKit, .linux, .windows, .android, .wasi, .openbsd] -let nonWASIPlatforms: [Platform] = allPlatforms.filter { $0 != .wasi } -let wasiPlatform: [Platform] = [.wasi] - -let package = Package( - name: "sqlite-nio", - platforms: [ - .macOS(.v10_15), - .iOS(.v13), - .watchOS(.v6), - .tvOS(.v13), - ], - products: [ - .library(name: "SQLiteNIO", targets: ["SQLiteNIO"]), - ], - traits: [ - .trait(name: "SwiftNIO", description: "Default backend built on SwiftNIO (EventLoopFuture)."), - .trait( - name: "EmbeddedWASI", - description: "NIO-free, Swift-Concurrency SQLite driver for Embedded Swift / WASI; drops swift-nio." - ), - .default(enabledTraits: ["SwiftNIO"]), - ], - dependencies: [ - .package(path: "../swift-nio"), - .package(path: "../swift-log"), - ], - targets: [ - .plugin( - name: "VendorSQLite", - capability: .command( - intent: .custom(verb: "vendor-sqlite", description: "Vendor SQLite"), - permissions: [ - .allowNetworkConnections(scope: .all(ports: [443]), reason: "Retrieve the latest build of SQLite"), - .writeToPackageDirectory(reason: "Update the vendored SQLite files"), - ] - ), - exclude: ["001-warnings-and-data-race.patch"] - ), - .target( - name: "CSQLite", - cSettings: sqliteCSettings - ), - .target( - name: "SQLiteNIO", - dependencies: [ - .target(name: "CSQLite"), - .product(name: "Logging", package: "swift-log"), - // SwiftNIO backend products — only when the SwiftNIO trait is enabled. - .product(name: "NIOCore", package: "swift-nio", condition: .when(traits: ["SwiftNIO"])), - .product(name: "NIOAsyncRuntime", package: "swift-nio", condition: .when(platforms: wasiPlatform, traits: ["SwiftNIO"])), - .product(name: "NIOPosix", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms, traits: ["SwiftNIO"])), - .product(name: "NIOFoundationCompat", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms, traits: ["SwiftNIO"])), - ], - swiftSettings: swiftSettings - ), - .testTarget( - name: "SQLiteNIOTests", - dependencies: [ - .target(name: "SQLiteNIO"), - ], - swiftSettings: swiftSettings - ), - ] -) - -var swiftSettings: [SwiftSetting] { [ - .enableUpcomingFeature("ExistentialAny"), - .enableUpcomingFeature("ConciseMagicFile"), - .enableUpcomingFeature("ForwardTrailingClosures"), - .enableUpcomingFeature("DisableOutwardActorInference"), - .enableExperimentalFeature("StrictConcurrency=complete"), -] } - -var sqliteCSettings: [CSetting] { [ - // Derived from sqlite3 version 3.43.0 - .define("SQLITE_DEFAULT_MEMSTATUS", to: "0"), - .define("SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS"), - .define("SQLITE_DQS", to: "0"), - .define("SQLITE_ENABLE_API_ARMOR", .when(configuration: .debug)), - .define("SQLITE_ENABLE_COLUMN_METADATA"), - .define("SQLITE_ENABLE_DBSTAT_VTAB"), - .define("SQLITE_ENABLE_FTS3"), - .define("SQLITE_ENABLE_FTS3_PARENTHESIS"), - .define("SQLITE_ENABLE_FTS3_TOKENIZER"), - .define("SQLITE_ENABLE_FTS4"), - .define("SQLITE_ENABLE_FTS5"), - .define("SQLITE_ENABLE_NULL_TRIM"), - .define("SQLITE_ENABLE_RTREE"), - .define("SQLITE_ENABLE_SESSION"), - .define("SQLITE_ENABLE_STMTVTAB"), - .define("SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION"), - .define("SQLITE_ENABLE_UNLOCK_NOTIFY"), - .define("SQLITE_MAX_VARIABLE_NUMBER", to: "250000"), - .define("SQLITE_LIKE_DOESNT_MATCH_BLOBS"), - .define("SQLITE_OMIT_COMPLETE"), - .define("SQLITE_OMIT_DEPRECATED"), - .define("SQLITE_OMIT_DESERIALIZE"), - .define("SQLITE_OMIT_GET_TABLE"), - .define("SQLITE_OMIT_LOAD_EXTENSION"), - .define("SQLITE_OMIT_PROGRESS_CALLBACK"), - .define("SQLITE_OMIT_SHARED_CACHE"), - .define("SQLITE_OMIT_TCL_VARIABLE"), - .define("SQLITE_OMIT_TRACE"), - .define("SQLITE_SECURE_DELETE"), - .define("SQLITE_THREADSAFE", to: "1", .when(platforms: nonWASIPlatforms)), - .define("SQLITE_THREADSAFE", to: "0", .when(platforms: wasiPlatform)), - .define("SQLITE_UNTESTABLE"), - .define("SQLITE_USE_URI"), -] } From 127e7405d0404eff76bf1a7750c648b3e7468c42 Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Tue, 16 Jun 2026 12:04:11 -0600 Subject: [PATCH 04/10] feat(embedded): NIO-free SQLiteData (ByteBuffer->[UInt8], gated Encodable) On WASI the blob case stores [UInt8] instead of ByteBuffer (NIOCore is elided). The Encodable conformance moves to a #if !hasFeature(Embedded) extension since Encoder is unavailable in Embedded Swift (still present on regular WASI). Co-Authored-By: Claude Opus 4.8 (1M context) --- Sources/SQLiteNIO/SQLiteData.swift | 45 ++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/Sources/SQLiteNIO/SQLiteData.swift b/Sources/SQLiteNIO/SQLiteData.swift index 8455a15..2d66234 100644 --- a/Sources/SQLiteNIO/SQLiteData.swift +++ b/Sources/SQLiteNIO/SQLiteData.swift @@ -1,5 +1,7 @@ import CSQLite +#if !os(WASI) import NIOCore +#endif #if _pointerBitWidth(_64) /// We use `Int` on 64-bit systems due to public API breakage concerns. @@ -17,7 +19,9 @@ public typealias SQLiteInt64 = Int64 // On 32-bit platforms, we want to use 64 b /// /// SQLite supports four data type "affinities" - INTEGER, REAL, TEXT, and BLOB - plus the `NULL` value, which has no /// innate affinity. -public enum SQLiteData: Equatable, Encodable, CustomStringConvertible, Sendable { +// `Encodable` is added via a conditional extension below: it relies on `Encoder`, which is +// unavailable in Embedded Swift. +public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { /// `INTEGER` affinity, represented in Swift by `Int`. case integer(SQLiteInt64) @@ -27,8 +31,12 @@ public enum SQLiteData: Equatable, Encodable, CustomStringConvertible, Sendable /// `TEXT` affinity, represented in Swift by `String`. case text(String) - /// `BLOB` affinity, represented in Swift by `ByteBuffer`. + /// `BLOB` affinity. Represented by SwiftNIO's `ByteBuffer`, or by `[UInt8]` on WASI (NIO-free). + #if os(WASI) + case blob([UInt8]) + #else case blob(ByteBuffer) + #endif /// A `NULL` value. case null @@ -95,6 +103,14 @@ public enum SQLiteData: Equatable, Encodable, CustomStringConvertible, Sendable /// Returns the data as a blob, if it has `BLOB` affinity. /// /// `INTEGER`, `REAL`, `TEXT`, and `NULL` values always return `nil`. + #if os(WASI) + public var blob: [UInt8]? { + switch self { + case .blob(let buffer): return buffer + case .integer, .float, .text, .null: return nil + } + } + #else public var blob: ByteBuffer? { switch self { case .blob(let buffer): @@ -103,6 +119,7 @@ public enum SQLiteData: Equatable, Encodable, CustomStringConvertible, Sendable return nil } } + #endif /// `true` if the value is `NULL`, `false` otherwise. public var isNull: Bool { @@ -117,7 +134,11 @@ public enum SQLiteData: Equatable, Encodable, CustomStringConvertible, Sendable // See `CustomStringConvertible.description`. public var description: String { switch self { + #if os(WASI) + case .blob(let data): return "<\(data.count) bytes>" + #else case .blob(let data): return "<\(data.readableBytes) bytes>" + #endif case .float(let float): return float.description case .integer(let int): return int.description case .null: return "null" @@ -125,19 +146,29 @@ public enum SQLiteData: Equatable, Encodable, CustomStringConvertible, Sendable } } - // See `Encodable.encode(to:)`. + // See `Encodable.encode(to:)`. Unavailable in Embedded Swift (no `Encoder`). + #if !hasFeature(Embedded) public func encode(to encoder: any Encoder) throws { var container = encoder.singleValueContainer() switch self { case .integer(let value): try container.encode(value) case .float(let value): try container.encode(value) case .text(let value): try container.encode(value) + #if os(WASI) + case .blob(let value): try container.encode(value) // [UInt8] encodes as raw bytes + #else case .blob(let value): try container.encode(Array(value.readableBytesView)) // N.B.: Don't use ByteBuffer's Codable conformance; it encodes as Base64, not raw bytes + #endif case .null: try container.encodeNil() } } + #endif } +#if !hasFeature(Embedded) +extension SQLiteData: Encodable {} +#endif + extension SQLiteData { /// Attempt to interpret an `sqlite3_value` as an equivalent ``SQLiteData``. init(sqliteValue: OpaquePointer) throws { @@ -157,10 +188,18 @@ extension SQLiteData { case SQLITE_BLOB: if let bytes = sqlite_nio_sqlite3_value_blob(sqliteValue) { let count = Int(sqlite_nio_sqlite3_value_bytes(sqliteValue)) + #if os(WASI) + self = .blob([UInt8](UnsafeRawBufferPointer(start: bytes, count: count))) // copy bytes + #else let buffer = ByteBuffer(bytes: UnsafeRawBufferPointer(start: bytes, count: count)) self = .blob(buffer) // copy bytes + #endif } else { + #if os(WASI) + self = .blob([]) + #else self = .blob(ByteBuffer()) + #endif } case let type: throw SQLiteCustomFunctionUnexpectedValueTypeError(type: type) From 8f8a63abc238713b6223c72780939d83091ff050 Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Tue, 16 Jun 2026 12:05:42 -0600 Subject: [PATCH 05/10] feat(embedded): NIO-free SQLiteStatement blob handling + gate NIO re-exports SQLiteStatement binds/reads blobs via [UInt8] (withUnsafeBytes / append) instead of ByteBuffer on WASI. Exports.swift re-exports only SwiftNIO types, so it's gated entirely on #if !os(WASI). Co-Authored-By: Claude Opus 4.8 (1M context) --- Sources/SQLiteNIO/Exports.swift | 15 +++------------ Sources/SQLiteNIO/SQLiteStatement.swift | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Sources/SQLiteNIO/Exports.swift b/Sources/SQLiteNIO/Exports.swift index 3dce47a..15131c4 100644 --- a/Sources/SQLiteNIO/Exports.swift +++ b/Sources/SQLiteNIO/Exports.swift @@ -1,21 +1,12 @@ +// These re-exports are all SwiftNIO types, which are elided on WASI (the NIO-free path uses +// Swift Concurrency + `[UInt8]` instead of EventLoop/ByteBuffer). +#if !os(WASI) @_documentation(visibility: internal) @_exported import struct NIOCore.ByteBuffer -// See SQLiteConnection.swift for why we gate on `os(WASI)` rather than -// `canImport(...)`: NIOAsyncRuntime is too greedy on macOS/Linux (its types -// require macOS 15+ and break older deployment targets), and NIOPosix is too -// greedy on WASI (the module imports as a partial stub but doesn't expose -// `MultiThreadedEventLoopGroup` / `NIOThreadPool` there). -#if os(WASI) -@_documentation(visibility: internal) @_exported import class NIOAsyncRuntime.AsyncThreadPool -#else @_documentation(visibility: internal) @_exported import class NIOPosix.NIOThreadPool -#endif @_documentation(visibility: internal) @_exported import protocol NIOCore.EventLoop @_documentation(visibility: internal) @_exported import protocol NIOCore.EventLoopGroup -#if os(WASI) -@_documentation(visibility: internal) @_exported import class NIOAsyncRuntime.AsyncEventLoopGroup -#else @_documentation(visibility: internal) @_exported import class NIOPosix.MultiThreadedEventLoopGroup #endif diff --git a/Sources/SQLiteNIO/SQLiteStatement.swift b/Sources/SQLiteNIO/SQLiteStatement.swift index 21912b4..9438d95 100644 --- a/Sources/SQLiteNIO/SQLiteStatement.swift +++ b/Sources/SQLiteNIO/SQLiteStatement.swift @@ -1,4 +1,6 @@ +#if !os(WASI) import NIOCore +#endif import CSQLite struct SQLiteStatement { @@ -40,9 +42,15 @@ struct SQLiteStatement { switch bind { case .blob(let value): + #if os(WASI) + ret = value.withUnsafeBytes { + sqlite_nio_sqlite3_bind_blob64(self.handle, i, $0.baseAddress, UInt64($0.count), SQLITE_TRANSIENT) + } + #else ret = value.withUnsafeReadableBytes { sqlite_nio_sqlite3_bind_blob64(self.handle, i, $0.baseAddress, UInt64($0.count), SQLITE_TRANSIENT) } + #endif case .float(let value): ret = sqlite_nio_sqlite3_bind_double(self.handle, i, value) case .integer(let value): @@ -102,12 +110,21 @@ struct SQLiteStatement { return .text(.init(cString: val)) case SQLITE_BLOB: let length = Int(sqlite_nio_sqlite3_column_bytes(self.handle, offset)) + #if os(WASI) + var bytes = [UInt8]() + bytes.reserveCapacity(length) + if let blobPointer = sqlite_nio_sqlite3_column_blob(self.handle, offset) { + bytes.append(contentsOf: UnsafeRawBufferPointer(start: blobPointer, count: length)) + } + return .blob(bytes) + #else var buffer = ByteBufferAllocator().buffer(capacity: length) - + if let blobPointer = sqlite_nio_sqlite3_column_blob(self.handle, offset) { buffer.writeBytes(UnsafeRawBufferPointer(start: blobPointer, count: length)) } return .blob(buffer) + #endif case SQLITE_NULL: return .null default: From df390080a10b5a130430db9d75dae4ac3dc50a52 Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Tue, 16 Jun 2026 12:19:41 -0600 Subject: [PATCH 06/10] feat(embedded): NIO-free async SQLiteConnection + SQLiteDatabase for WASI Gate the NIO EventLoopFuture/NIOThreadPool connection and the hook subsystem behind #if !os(WASI); add SQLiteConnection+WASI.swift, a Swift-Concurrency connection over CSQLite (blocking calls run inline since wasm is single-threaded; no EventLoop/threadpool/hooks/pool). The WASI SQLiteDatabase protocol has only non-generic requirements (logger + async query) so it remains usable as 'any SQLiteDatabase' under Embedded Swift; withConnection stays on the concrete connection. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../SQLiteNIO/SQLiteConnection+Hooks.swift | 3 + Sources/SQLiteNIO/SQLiteConnection+WASI.swift | 148 ++++++++++++++++++ Sources/SQLiteNIO/SQLiteConnection.swift | 3 + Sources/SQLiteNIO/SQLiteDatabase.swift | 69 ++++++++ 4 files changed, 223 insertions(+) create mode 100644 Sources/SQLiteNIO/SQLiteConnection+WASI.swift diff --git a/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift b/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift index 93977e5..10f736b 100644 --- a/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift +++ b/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift @@ -1,3 +1,4 @@ +#if !os(WASI) // EMBEDDED-WASI: NIO-based connection; the WASI build uses SQLiteConnection+WASI.swift import Foundation import NIOConcurrencyHelpers import NIOCore @@ -865,3 +866,5 @@ extension SQLiteConnection { self.observerBuckets.withLockedValue { $0 = .init() } } } + +#endif // !os(WASI) diff --git a/Sources/SQLiteNIO/SQLiteConnection+WASI.swift b/Sources/SQLiteNIO/SQLiteConnection+WASI.swift new file mode 100644 index 0000000..d3a97af --- /dev/null +++ b/Sources/SQLiteNIO/SQLiteConnection+WASI.swift @@ -0,0 +1,148 @@ +#if os(WASI) +import CSQLite +import Logging + +/// A wrapper for the `OpaquePointer` used to represent an open `sqlite3` handle. +/// +/// On WASI the runtime is single-threaded, so no thread pool / locking is involved; the +/// `@unchecked Sendable` is justified the same way as the SwiftNIO build (serialized SQLite, +/// `SQLITE_OPEN_FULLMUTEX`), and the handle is only mutated to `nil` on close. +final class SQLiteConnectionHandle: @unchecked Sendable { + var raw: OpaquePointer? + + init(_ raw: OpaquePointer?) { + self.raw = raw + } +} + +/// A single open connection to an SQLite database (WASI / Embedded build). +/// +/// This is the NIO-free variant: it exposes a Swift-Concurrency (`async`/`await`) API over CSQLite +/// with no `EventLoopFuture`, `EventLoopGroup`, or `NIOThreadPool`. Because WebAssembly is +/// single-threaded, the blocking libsqlite3 calls run inline on the calling task. The observable +/// hook API and connection pooling are not available on this build. +public final class SQLiteConnection: SQLiteDatabase, Sendable { + /// The possible storage types for an SQLite database. + public enum Storage: Equatable, Sendable { + /// An SQLite database stored entirely in memory. + case memory + + /// An SQLite database stored in a file at the specified path. + case file(path: String) + } + + /// Return the version of the embedded libsqlite3 as a 32-bit integer value. + public static func libraryVersion() -> Int32 { + sqlite_nio_sqlite3_libversion_number() + } + + /// Return the version of the embedded libsqlite3 as a string. + public static func libraryVersionString() -> String { + String(cString: sqlite_nio_sqlite3_libversion()) + } + + // See `SQLiteDatabase.logger`. + public let logger: Logger + + /// The underlying `sqlite3` connection handle. + let handle: SQLiteConnectionHandle + + /// Initialize a new ``SQLiteConnection``. Internal use only. + private init(handle: OpaquePointer?, logger: Logger) { + self.handle = .init(handle) + self.logger = logger + } + + /// Returns the most recent error message from the connection as a string. + var errorMessage: String? { + sqlite_nio_sqlite3_errmsg(self.handle.raw).map { String(cString: $0) } + } + + /// `false` if the connection is valid, `true` if not. + public var isClosed: Bool { + self.handle.raw == nil + } + + /// Open a new connection to an SQLite database. + /// + /// - Parameters: + /// - storage: Specifies the location of the database for the connection. See ``Storage`` for details. + /// - logger: The logger used by the connection. Defaults to a new `Logger`. + /// - Returns: A new connection object. + public static func open( + storage: Storage = .memory, + logger: Logger = .init(label: "codes.vapor.sqlite") + ) async throws -> SQLiteConnection { + let path: String + switch storage { + case .memory: path = ":memory:" + case .file(let file): path = file + } + + var handle: OpaquePointer? + let openOptions = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_URI | SQLITE_OPEN_EXRESCODE + let openRet = sqlite_nio_sqlite3_open_v2(path, &handle, openOptions, nil) + guard openRet == SQLITE_OK else { + throw SQLiteError(reason: .init(statusCode: openRet), message: "Failed to open to SQLite database at \(path)") + } + + let busyRet = sqlite_nio_sqlite3_busy_handler(handle, { _, _ in 1 }, nil) + guard busyRet == SQLITE_OK else { + sqlite_nio_sqlite3_close(handle) + throw SQLiteError(reason: .init(statusCode: busyRet), message: "Failed to set busy handler for SQLite database at \(path)") + } + + logger.debug("Connected to sqlite database", metadata: ["path": .string(path)]) + return SQLiteConnection(handle: handle, logger: logger) + } + + /// Returns the last value generated by auto-increment functionality on this database. + public func lastAutoincrementID() async throws -> Int { + numericCast(sqlite_nio_sqlite3_last_insert_rowid(self.handle.raw)) + } + + /// Run the provided closure with this connection. + public func withConnection( + _ closure: @escaping @Sendable (SQLiteConnection) async throws -> T + ) async throws -> T { + try await closure(self) + } + + // See `SQLiteDatabase.query(_:_:logger:_:)`. + public func query( + _ query: String, + _ binds: [SQLiteData], + logger: Logger, + _ onRow: @escaping @Sendable (SQLiteRow) -> Void + ) async throws { + var statement = try SQLiteStatement(query: query, on: self) + let columns = try statement.columns() + try statement.bind(binds) + while let row = try statement.nextRow(for: columns) { + onRow(row) + } + } + + /// Close the connection and invalidate its handle. + public func close() async throws { + sqlite_nio_sqlite3_close(self.handle.raw) + self.handle.raw = nil + } + + /// Install the provided ``SQLiteCustomFunction`` on the connection. + public func install(customFunction: SQLiteCustomFunction) async throws { + self.logger.trace("Adding custom function \(customFunction.name)") + try customFunction.install(in: self) + } + + /// Uninstall the provided ``SQLiteCustomFunction`` from the connection. + public func uninstall(customFunction: SQLiteCustomFunction) async throws { + self.logger.trace("Removing custom function \(customFunction.name)") + try customFunction.uninstall(in: self) + } + + deinit { + assert(self.handle.raw == nil, "SQLiteConnection was not closed before deinitializing") + } +} +#endif // os(WASI) diff --git a/Sources/SQLiteNIO/SQLiteConnection.swift b/Sources/SQLiteNIO/SQLiteConnection.swift index 14caf0f..3ce654e 100644 --- a/Sources/SQLiteNIO/SQLiteConnection.swift +++ b/Sources/SQLiteNIO/SQLiteConnection.swift @@ -1,3 +1,4 @@ +#if !os(WASI) // EMBEDDED-WASI: NIO-based connection; the WASI build uses SQLiteConnection+WASI.swift import NIOConcurrencyHelpers import NIOCore // Use NIOPosix on every host platform that supports it (macOS, Linux, etc.) @@ -457,3 +458,5 @@ extension SQLiteConnection { } } } + +#endif // !os(WASI) diff --git a/Sources/SQLiteNIO/SQLiteDatabase.swift b/Sources/SQLiteNIO/SQLiteDatabase.swift index 8fc3518..702aebf 100644 --- a/Sources/SQLiteNIO/SQLiteDatabase.swift +++ b/Sources/SQLiteNIO/SQLiteDatabase.swift @@ -1,3 +1,4 @@ +#if !os(WASI) // EMBEDDED-WASI: NIO/EventLoopFuture protocol; WASI uses the async protocol below import NIOCore import CSQLite import Logging @@ -195,3 +196,71 @@ private struct SQLiteDatabaseCustomLogger: SQLiteDatabase { Self(database: self.database, logger: logger) } } + +#endif // !os(WASI) + +#if os(WASI) +import CSQLite +import Logging + +/// NIO-free (`async`/`await`) variant of ``SQLiteDatabase`` for the WASI / Embedded build. +/// +/// The protocol deliberately has only non-generic requirements so it remains usable as an +/// existential (`any SQLiteDatabase`) under Embedded Swift, which cannot place a generic method in a +/// protocol witness table. `withConnection(_:)` is therefore provided on the concrete +/// ``SQLiteConnection`` rather than as a protocol requirement. +public protocol SQLiteDatabase: Sendable { + /// The logger used by the connection. + var logger: Logger { get } + + /// Execute a query, invoking `onRow` for each result row. + func query( + _ query: String, + _ binds: [SQLiteData], + logger: Logger, + _ onRow: @escaping @Sendable (SQLiteRow) -> Void + ) async throws +} + +extension SQLiteDatabase { + /// Convenience: execute a query using the database's own logger. + public func query( + _ query: String, + _ binds: [SQLiteData] = [], + _ onRow: @escaping @Sendable (SQLiteRow) -> Void + ) async throws { + try await self.query(query, binds, logger: self.logger, onRow) + } + + /// Execute a query and collect the result rows. + public func query(_ query: String, _ binds: [SQLiteData] = []) async throws -> [SQLiteRow] { + nonisolated(unsafe) var rows: [SQLiteRow] = [] + try await self.query(query, binds) { rows.append($0) } + return rows + } + + /// Return a database that logs to `logger`, forwarding everything else to `self`. + public func logging(to logger: Logger) -> any SQLiteDatabase { + SQLiteDatabaseCustomLogger(database: self, logger: logger) + } +} + +/// Replaces the `Logger` of an existing ``SQLiteDatabase`` while forwarding queries to the original. +private struct SQLiteDatabaseCustomLogger: SQLiteDatabase { + let database: D + let logger: Logger + + func query( + _ query: String, + _ binds: [SQLiteData], + logger: Logger, + _ onRow: @escaping @Sendable (SQLiteRow) -> Void + ) async throws { + try await self.database.query(query, binds, logger: logger, onRow) + } + + func logging(to logger: Logger) -> any SQLiteDatabase { + Self(database: self.database, logger: logger) + } +} +#endif // os(WASI) From b921581e7fa8baf2c8717341be06d99a5aa021da Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Tue, 16 Jun 2026 12:19:41 -0600 Subject: [PATCH 07/10] feat(embedded): drop Foundation/ByteBuffer from SQLiteNIO value + error types on WASI SQLiteError: LocalizedError moved to a #if !os(WASI) extension. SQLiteDataConvertible: ByteBuffer/Data/Date conformances gated, [UInt8] blob conformance added for WASI. SQLiteDataType: gate the any-Encodable serialize (Embedded). SQLiteRow/SQLiteCustomFunction: gate reflection-based Array.description, any-Error interpolation, and ByteBuffer blob result handling. Co-Authored-By: Claude Opus 4.8 (1M context) --- Sources/SQLiteNIO/SQLiteCustomFunction.swift | 10 +++++++++ Sources/SQLiteNIO/SQLiteDataConvertible.swift | 22 +++++++++++++++++++ Sources/SQLiteNIO/SQLiteDataType.swift | 3 +++ Sources/SQLiteNIO/SQLiteError.swift | 12 +++++++++- Sources/SQLiteNIO/SQLiteRow.swift | 5 +++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Sources/SQLiteNIO/SQLiteCustomFunction.swift b/Sources/SQLiteNIO/SQLiteCustomFunction.swift index 4f18b4e..1b5eab7 100644 --- a/Sources/SQLiteNIO/SQLiteCustomFunction.swift +++ b/Sources/SQLiteNIO/SQLiteCustomFunction.swift @@ -300,9 +300,15 @@ public final class SQLiteCustomFunction: Hashable { case .text(let string): sqlite_nio_sqlite3_result_text(sqliteContext, string, -1, SQLITE_TRANSIENT) case .blob(let value): + #if os(WASI) + value.withUnsafeBytes { pointer in + sqlite_nio_sqlite3_result_blob(sqliteContext, pointer.baseAddress, Int32(value.count), SQLITE_TRANSIENT) + } + #else value.withUnsafeReadableBytes { pointer in sqlite_nio_sqlite3_result_blob(sqliteContext, pointer.baseAddress, Int32(value.readableBytes), SQLITE_TRANSIENT) } + #endif } } @@ -311,7 +317,11 @@ public final class SQLiteCustomFunction: Hashable { sqlite_nio_sqlite3_result_error(sqliteContext, error.message, -1) sqlite_nio_sqlite3_result_error_code(sqliteContext, error.reason.statusCode) } else { + #if hasFeature(Embedded) + sqlite_nio_sqlite3_result_error(sqliteContext, "custom function error", -1) // `any Error` interpolation needs reflection + #else sqlite_nio_sqlite3_result_error(sqliteContext, "\(error)", -1) + #endif } } } diff --git a/Sources/SQLiteNIO/SQLiteDataConvertible.swift b/Sources/SQLiteNIO/SQLiteDataConvertible.swift index abbe578..62506f4 100644 --- a/Sources/SQLiteNIO/SQLiteDataConvertible.swift +++ b/Sources/SQLiteNIO/SQLiteDataConvertible.swift @@ -1,6 +1,10 @@ +// ByteBuffer (NIOCore), Data/Date (Foundation), and the NIOFoundationCompat bridges are all elided +// on WASI; see the `#if os(WASI)` / `#if !os(WASI)` conformances below. +#if !os(WASI) import NIOCore import NIOFoundationCompat import Foundation +#endif public protocol SQLiteDataConvertible { init?(sqliteData: SQLiteData) @@ -74,6 +78,20 @@ extension Float: SQLiteDataConvertible { } } +#if os(WASI) +extension [UInt8]: SQLiteDataConvertible { + public init?(sqliteData: SQLiteData) { + guard case .blob(let value) = sqliteData else { + return nil + } + self = value + } + + public var sqliteData: SQLiteData? { + .blob(self) + } +} +#else extension ByteBuffer: SQLiteDataConvertible { public init?(sqliteData: SQLiteData) { guard case .blob(let value) = sqliteData else { @@ -99,6 +117,7 @@ extension Data: SQLiteDataConvertible { .blob(.init(data: self)) } } +#endif extension Bool: SQLiteDataConvertible { public init?(sqliteData: SQLiteData) { @@ -113,6 +132,8 @@ extension Bool: SQLiteDataConvertible { } } +// Date conversions rely on Foundation (`Date`, `ISO8601DateFormatter`), unavailable on WASI. +#if !os(WASI) extension Date: SQLiteDataConvertible { public init?(sqliteData: SQLiteData) { let value: Double @@ -171,3 +192,4 @@ var dateFormatter: ISO8601DateFormatter { ] return formatter } +#endif // !os(WASI) diff --git a/Sources/SQLiteNIO/SQLiteDataType.swift b/Sources/SQLiteNIO/SQLiteDataType.swift index a899699..db5bba3 100644 --- a/Sources/SQLiteNIO/SQLiteDataType.swift +++ b/Sources/SQLiteNIO/SQLiteDataType.swift @@ -16,6 +16,8 @@ public enum SQLiteDataType { /// `NULL`. case null + // `any Encodable` is unavailable in Embedded Swift; this type is deprecated/unused anyway. + #if !hasFeature(Embedded) public func serialize(_ binds: inout [any Encodable]) -> String { switch self { case .integer: return "INTEGER" @@ -25,4 +27,5 @@ public enum SQLiteDataType { case .null: return "NULL" } } + #endif } diff --git a/Sources/SQLiteNIO/SQLiteError.swift b/Sources/SQLiteNIO/SQLiteError.swift index 57deaa8..3b93c56 100644 --- a/Sources/SQLiteNIO/SQLiteError.swift +++ b/Sources/SQLiteNIO/SQLiteError.swift @@ -1,7 +1,17 @@ import CSQLite +// Foundation provides `LocalizedError`; unavailable on WASI (`errorDescription` is kept as a plain +// property there). +#if !os(WASI) import Foundation +#endif -public struct SQLiteError: Error, CustomStringConvertible, LocalizedError { +// `LocalizedError` is Foundation-only; declared via a gated extension (the `errorDescription` +// witness lives in the struct body and is simply unused on WASI). +#if !os(WASI) +extension SQLiteError: LocalizedError {} +#endif + +public struct SQLiteError: Error, CustomStringConvertible { public let reason: Reason public let message: String diff --git a/Sources/SQLiteNIO/SQLiteRow.swift b/Sources/SQLiteNIO/SQLiteRow.swift index abddcad..f4bfe50 100644 --- a/Sources/SQLiteNIO/SQLiteRow.swift +++ b/Sources/SQLiteNIO/SQLiteRow.swift @@ -25,7 +25,12 @@ public struct SQLiteRow: CustomStringConvertible, Sendable { } public var description: String { + #if hasFeature(Embedded) + // `Array.description` uses reflection, unavailable in Embedded Swift. + "[" + self.columns.map { $0.description }.joined(separator: ", ") + "]" + #else self.columns.description + #endif } } From 8fd719a5e4308f2f6717b4195dea217cb4e1d6a4 Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Mon, 6 Jul 2026 12:23:47 -0600 Subject: [PATCH 08/10] refactor(embedded): make the NIO-free path Embedded-only (flavor-safe for khasm) The embedded port previously selected the NIO-free Swift-Concurrency driver with `#if os(WASI)` and dropped SwiftNIO for every WASI build. khasm's shipping regular-wasm flavor, however, runs SQLiteNIO on WASI with NIO (NIOAsyncRuntime event loops, ByteBuffer blobs), so the wasi-wide gate would have changed the shipping data path. - Source gates `#if os(WASI)` -> `#if hasFeature(Embedded)` for every gate the embedded port introduced (regular WASI is restored to the feat/khasmPAL-2026 base behavior: NIO protocol, ByteBuffer blob, NIOAsyncRuntime substitution). Pre-existing os(WASI) gates inside the NIO connection are untouched. - Exports.swift: base NIO re-exports restored, wrapped in `#if !hasFeature(Embedded)`. - Package.swift: base NIO products restored (incl. NIOAsyncRuntime on wasi); the whole NIO stack is dropped only with KHASM_EMBEDDED=1 (manifests cannot see hasFeature; matches khasm's manifest gating). Verified: embedded SDK build (KHASM_EMBEDDED=1) green; native green. Co-Authored-By: Claude Fable 5 --- Package.swift | 28 +++++++++++++------ Sources/SQLiteNIO/Exports.swift | 19 +++++++++++-- .../SQLiteNIO/SQLiteConnection+Hooks.swift | 4 +-- Sources/SQLiteNIO/SQLiteConnection+WASI.swift | 4 +-- Sources/SQLiteNIO/SQLiteConnection.swift | 4 +-- Sources/SQLiteNIO/SQLiteCustomFunction.swift | 2 +- Sources/SQLiteNIO/SQLiteData.swift | 14 +++++----- Sources/SQLiteNIO/SQLiteDataConvertible.swift | 10 +++---- Sources/SQLiteNIO/SQLiteDatabase.swift | 8 +++--- Sources/SQLiteNIO/SQLiteError.swift | 4 +-- Sources/SQLiteNIO/SQLiteStatement.swift | 6 ++-- 11 files changed, 64 insertions(+), 39 deletions(-) diff --git a/Package.swift b/Package.swift index 5584766..3e2171d 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,5 @@ // swift-tools-version:5.10 +import class Foundation.ProcessInfo import PackageDescription /// This list matches the [supported platforms on the Swift 5.10 release of SPM](https://github.com/swiftlang/swift-package-manager/blob/release/5.10/Sources/PackageDescription/SupportedPlatforms.swift#L34-L71) @@ -7,6 +8,15 @@ let allPlatforms: [Platform] = [.macOS, .macCatalyst, .iOS, .tvOS, .watchOS, .vi let nonWASIPlatforms: [Platform] = allPlatforms.filter { $0 != .wasi } let wasiPlatform: [Platform] = [.wasi] +// Embedded-wasm port (see /Users/scottm/git/c34/khasm/EMBEDDED_PORT_PLAN.md): with +// KHASM_EMBEDDED=1 the SwiftNIO stack is dropped entirely — the Embedded build uses the +// NIO-free, Swift-Concurrency driver over CSQLite gated in source with +// `#if hasFeature(Embedded)`. `hasFeature` is not available in manifests (they run +// host-side), hence the env-var conditional, matching khasm's own manifest gating. +// Regular (non-embedded) builds — including regular WASI — keep the NIO stack exactly +// as on the feat/khasmPAL-2026 base. +let khasmEmbedded = ProcessInfo.processInfo.environment["KHASM_EMBEDDED"] == "1" + let package = Package( name: "sqlite-nio", platforms: [ @@ -44,14 +54,16 @@ let package = Package( dependencies: [ .target(name: "CSQLite"), .product(name: "Logging", package: "swift-log"), - // The SwiftNIO stack is elided on WASI (normal + Embedded); .when(platforms:) is - // target-evaluated, so these aren't built when cross-compiling to wasm32-unknown-wasip1. - // The WASI path is a NIO-free, Swift-Concurrency driver over CSQLite, gated in source - // with `#if os(WASI)`. - .product(name: "NIOCore", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms)), - .product(name: "NIOPosix", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms)), - .product(name: "NIOFoundationCompat", package: "swift-nio", condition: .when(platforms: nonWASIPlatforms)), - ], + ] + (khasmEmbedded ? [] : [ + // The SwiftNIO stack is dropped on the Embedded build (KHASM_EMBEDDED=1, see the + // note at the top); the Embedded path is a NIO-free, Swift-Concurrency driver over + // CSQLite, gated in source with `#if hasFeature(Embedded)`. All regular builds + // (native + regular WASI via NIOAsyncRuntime) keep the NIO stack. + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOAsyncRuntime", package: "swift-nio", condition: .when(platforms: wasiPlatform)), + .product(name: "NIOPosix", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + ]), swiftSettings: swiftSettings ), .testTarget( diff --git a/Sources/SQLiteNIO/Exports.swift b/Sources/SQLiteNIO/Exports.swift index 15131c4..9f144f3 100644 --- a/Sources/SQLiteNIO/Exports.swift +++ b/Sources/SQLiteNIO/Exports.swift @@ -1,12 +1,25 @@ -// These re-exports are all SwiftNIO types, which are elided on WASI (the NIO-free path uses -// Swift Concurrency + `[UInt8]` instead of EventLoop/ByteBuffer). -#if !os(WASI) +// These re-exports are all SwiftNIO types, which are elided on the Embedded build (the NIO-free +// Embedded path uses Swift Concurrency + `[UInt8]` instead of EventLoop/ByteBuffer). +#if !hasFeature(Embedded) @_documentation(visibility: internal) @_exported import struct NIOCore.ByteBuffer +// See SQLiteConnection.swift for why we gate on `os(WASI)` rather than +// `canImport(...)`: NIOAsyncRuntime is too greedy on macOS/Linux (its types +// require macOS 15+ and break older deployment targets), and NIOPosix is too +// greedy on WASI (the module imports as a partial stub but doesn't expose +// `MultiThreadedEventLoopGroup` / `NIOThreadPool` there). +#if os(WASI) +@_documentation(visibility: internal) @_exported import class NIOAsyncRuntime.AsyncThreadPool +#else @_documentation(visibility: internal) @_exported import class NIOPosix.NIOThreadPool +#endif @_documentation(visibility: internal) @_exported import protocol NIOCore.EventLoop @_documentation(visibility: internal) @_exported import protocol NIOCore.EventLoopGroup +#if os(WASI) +@_documentation(visibility: internal) @_exported import class NIOAsyncRuntime.AsyncEventLoopGroup +#else @_documentation(visibility: internal) @_exported import class NIOPosix.MultiThreadedEventLoopGroup #endif +#endif // !hasFeature(Embedded) diff --git a/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift b/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift index 10f736b..ab955eb 100644 --- a/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift +++ b/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift @@ -1,4 +1,4 @@ -#if !os(WASI) // EMBEDDED-WASI: NIO-based connection; the WASI build uses SQLiteConnection+WASI.swift +#if !hasFeature(Embedded) // EMBEDDED-WASI: NIO-based connection; the WASI build uses SQLiteConnection+WASI.swift import Foundation import NIOConcurrencyHelpers import NIOCore @@ -867,4 +867,4 @@ extension SQLiteConnection { } } -#endif // !os(WASI) +#endif // !hasFeature(Embedded) diff --git a/Sources/SQLiteNIO/SQLiteConnection+WASI.swift b/Sources/SQLiteNIO/SQLiteConnection+WASI.swift index d3a97af..6b30a00 100644 --- a/Sources/SQLiteNIO/SQLiteConnection+WASI.swift +++ b/Sources/SQLiteNIO/SQLiteConnection+WASI.swift @@ -1,4 +1,4 @@ -#if os(WASI) +#if hasFeature(Embedded) import CSQLite import Logging @@ -145,4 +145,4 @@ public final class SQLiteConnection: SQLiteDatabase, Sendable { assert(self.handle.raw == nil, "SQLiteConnection was not closed before deinitializing") } } -#endif // os(WASI) +#endif // hasFeature(Embedded) diff --git a/Sources/SQLiteNIO/SQLiteConnection.swift b/Sources/SQLiteNIO/SQLiteConnection.swift index 3ce654e..5de3bbf 100644 --- a/Sources/SQLiteNIO/SQLiteConnection.swift +++ b/Sources/SQLiteNIO/SQLiteConnection.swift @@ -1,4 +1,4 @@ -#if !os(WASI) // EMBEDDED-WASI: NIO-based connection; the WASI build uses SQLiteConnection+WASI.swift +#if !hasFeature(Embedded) // EMBEDDED: NIO-based connection; the Embedded build uses SQLiteConnection+WASI.swift import NIOConcurrencyHelpers import NIOCore // Use NIOPosix on every host platform that supports it (macOS, Linux, etc.) @@ -459,4 +459,4 @@ extension SQLiteConnection { } } -#endif // !os(WASI) +#endif // !hasFeature(Embedded) diff --git a/Sources/SQLiteNIO/SQLiteCustomFunction.swift b/Sources/SQLiteNIO/SQLiteCustomFunction.swift index 1b5eab7..ada926f 100644 --- a/Sources/SQLiteNIO/SQLiteCustomFunction.swift +++ b/Sources/SQLiteNIO/SQLiteCustomFunction.swift @@ -300,7 +300,7 @@ public final class SQLiteCustomFunction: Hashable { case .text(let string): sqlite_nio_sqlite3_result_text(sqliteContext, string, -1, SQLITE_TRANSIENT) case .blob(let value): - #if os(WASI) + #if hasFeature(Embedded) value.withUnsafeBytes { pointer in sqlite_nio_sqlite3_result_blob(sqliteContext, pointer.baseAddress, Int32(value.count), SQLITE_TRANSIENT) } diff --git a/Sources/SQLiteNIO/SQLiteData.swift b/Sources/SQLiteNIO/SQLiteData.swift index 2d66234..ba5dd6a 100644 --- a/Sources/SQLiteNIO/SQLiteData.swift +++ b/Sources/SQLiteNIO/SQLiteData.swift @@ -1,5 +1,5 @@ import CSQLite -#if !os(WASI) +#if !hasFeature(Embedded) import NIOCore #endif @@ -32,7 +32,7 @@ public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { case text(String) /// `BLOB` affinity. Represented by SwiftNIO's `ByteBuffer`, or by `[UInt8]` on WASI (NIO-free). - #if os(WASI) + #if hasFeature(Embedded) case blob([UInt8]) #else case blob(ByteBuffer) @@ -103,7 +103,7 @@ public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { /// Returns the data as a blob, if it has `BLOB` affinity. /// /// `INTEGER`, `REAL`, `TEXT`, and `NULL` values always return `nil`. - #if os(WASI) + #if hasFeature(Embedded) public var blob: [UInt8]? { switch self { case .blob(let buffer): return buffer @@ -134,7 +134,7 @@ public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { // See `CustomStringConvertible.description`. public var description: String { switch self { - #if os(WASI) + #if hasFeature(Embedded) case .blob(let data): return "<\(data.count) bytes>" #else case .blob(let data): return "<\(data.readableBytes) bytes>" @@ -154,7 +154,7 @@ public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { case .integer(let value): try container.encode(value) case .float(let value): try container.encode(value) case .text(let value): try container.encode(value) - #if os(WASI) + #if hasFeature(Embedded) case .blob(let value): try container.encode(value) // [UInt8] encodes as raw bytes #else case .blob(let value): try container.encode(Array(value.readableBytesView)) // N.B.: Don't use ByteBuffer's Codable conformance; it encodes as Base64, not raw bytes @@ -188,14 +188,14 @@ extension SQLiteData { case SQLITE_BLOB: if let bytes = sqlite_nio_sqlite3_value_blob(sqliteValue) { let count = Int(sqlite_nio_sqlite3_value_bytes(sqliteValue)) - #if os(WASI) + #if hasFeature(Embedded) self = .blob([UInt8](UnsafeRawBufferPointer(start: bytes, count: count))) // copy bytes #else let buffer = ByteBuffer(bytes: UnsafeRawBufferPointer(start: bytes, count: count)) self = .blob(buffer) // copy bytes #endif } else { - #if os(WASI) + #if hasFeature(Embedded) self = .blob([]) #else self = .blob(ByteBuffer()) diff --git a/Sources/SQLiteNIO/SQLiteDataConvertible.swift b/Sources/SQLiteNIO/SQLiteDataConvertible.swift index 62506f4..3e1fd1e 100644 --- a/Sources/SQLiteNIO/SQLiteDataConvertible.swift +++ b/Sources/SQLiteNIO/SQLiteDataConvertible.swift @@ -1,6 +1,6 @@ // ByteBuffer (NIOCore), Data/Date (Foundation), and the NIOFoundationCompat bridges are all elided -// on WASI; see the `#if os(WASI)` / `#if !os(WASI)` conformances below. -#if !os(WASI) +// on WASI; see the `#if hasFeature(Embedded)` / `#if !hasFeature(Embedded)` conformances below. +#if !hasFeature(Embedded) import NIOCore import NIOFoundationCompat import Foundation @@ -78,7 +78,7 @@ extension Float: SQLiteDataConvertible { } } -#if os(WASI) +#if hasFeature(Embedded) extension [UInt8]: SQLiteDataConvertible { public init?(sqliteData: SQLiteData) { guard case .blob(let value) = sqliteData else { @@ -133,7 +133,7 @@ extension Bool: SQLiteDataConvertible { } // Date conversions rely on Foundation (`Date`, `ISO8601DateFormatter`), unavailable on WASI. -#if !os(WASI) +#if !hasFeature(Embedded) extension Date: SQLiteDataConvertible { public init?(sqliteData: SQLiteData) { let value: Double @@ -192,4 +192,4 @@ var dateFormatter: ISO8601DateFormatter { ] return formatter } -#endif // !os(WASI) +#endif // !hasFeature(Embedded) diff --git a/Sources/SQLiteNIO/SQLiteDatabase.swift b/Sources/SQLiteNIO/SQLiteDatabase.swift index 702aebf..12bb2b7 100644 --- a/Sources/SQLiteNIO/SQLiteDatabase.swift +++ b/Sources/SQLiteNIO/SQLiteDatabase.swift @@ -1,4 +1,4 @@ -#if !os(WASI) // EMBEDDED-WASI: NIO/EventLoopFuture protocol; WASI uses the async protocol below +#if !hasFeature(Embedded) // EMBEDDED-WASI: NIO/EventLoopFuture protocol; WASI uses the async protocol below import NIOCore import CSQLite import Logging @@ -197,9 +197,9 @@ private struct SQLiteDatabaseCustomLogger: SQLiteDatabase { } } -#endif // !os(WASI) +#endif // !hasFeature(Embedded) -#if os(WASI) +#if hasFeature(Embedded) import CSQLite import Logging @@ -263,4 +263,4 @@ private struct SQLiteDatabaseCustomLogger: SQLiteDatabase { Self(database: self.database, logger: logger) } } -#endif // os(WASI) +#endif // hasFeature(Embedded) diff --git a/Sources/SQLiteNIO/SQLiteError.swift b/Sources/SQLiteNIO/SQLiteError.swift index 3b93c56..4310943 100644 --- a/Sources/SQLiteNIO/SQLiteError.swift +++ b/Sources/SQLiteNIO/SQLiteError.swift @@ -1,13 +1,13 @@ import CSQLite // Foundation provides `LocalizedError`; unavailable on WASI (`errorDescription` is kept as a plain // property there). -#if !os(WASI) +#if !hasFeature(Embedded) import Foundation #endif // `LocalizedError` is Foundation-only; declared via a gated extension (the `errorDescription` // witness lives in the struct body and is simply unused on WASI). -#if !os(WASI) +#if !hasFeature(Embedded) extension SQLiteError: LocalizedError {} #endif diff --git a/Sources/SQLiteNIO/SQLiteStatement.swift b/Sources/SQLiteNIO/SQLiteStatement.swift index 9438d95..ac1eece 100644 --- a/Sources/SQLiteNIO/SQLiteStatement.swift +++ b/Sources/SQLiteNIO/SQLiteStatement.swift @@ -1,4 +1,4 @@ -#if !os(WASI) +#if !hasFeature(Embedded) import NIOCore #endif import CSQLite @@ -42,7 +42,7 @@ struct SQLiteStatement { switch bind { case .blob(let value): - #if os(WASI) + #if hasFeature(Embedded) ret = value.withUnsafeBytes { sqlite_nio_sqlite3_bind_blob64(self.handle, i, $0.baseAddress, UInt64($0.count), SQLITE_TRANSIENT) } @@ -110,7 +110,7 @@ struct SQLiteStatement { return .text(.init(cString: val)) case SQLITE_BLOB: let length = Int(sqlite_nio_sqlite3_column_bytes(self.handle, offset)) - #if os(WASI) + #if hasFeature(Embedded) var bytes = [UInt8]() bytes.reserveCapacity(length) if let blobPointer = sqlite_nio_sqlite3_column_blob(self.handle, offset) { From 7f41021ae9e141a9e078e46f337eb41cc3426f4d Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Tue, 7 Jul 2026 15:47:25 -0600 Subject: [PATCH 09/10] build(embedded): reference swift-nio via the PassiveLogic fork URL, not the local clone Transitive path deps override URL declarations by identity in a consuming root graph (khasm), and the local ../swift-nio clone carries the vestigial Option-1/2 embedded-NIO commits whose os(WASI) gates would change regular-WASI NIO behavior. Under KHASM_EMBEDDED=1 the NIO products are dropped, so the embedded build never compiles NIO from either source. Co-Authored-By: Claude Fable 5 --- Package.swift | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 3e2171d..d23ae6b 100644 --- a/Package.swift +++ b/Package.swift @@ -29,8 +29,14 @@ let package = Package( .library(name: "SQLiteNIO", targets: ["SQLiteNIO"]), ], dependencies: [ - // Local clones for the embedded-wasm port (see /Users/scottm/git/c34/EMBEDDED_WASM_NOTES.md) - .package(path: "../swift-nio"), + // swift-nio stays on the PassiveLogic fork URL (feat/khasmPAL-2026, as on the base + // branch) rather than the local ../swift-nio clone: that clone carries the vestigial + // Option-1/2 embedded-NIO commits whose os(WASI) gates would change regular-WASI NIO + // behavior, and transitive *path* deps override URL declarations by identity in a + // consuming root graph (khasm). Under KHASM_EMBEDDED=1 NIO products are dropped, so + // the embedded build never compiles NIO from either source. + .package(url: "https://github.com/PassiveLogic/swift-nio.git", branch: "feat/khasmPAL-2026"), + // Local embedded-ported clone (see /Users/scottm/git/c34/EMBEDDED_WASM_NOTES.md). .package(path: "../swift-log"), ], targets: [ From 3591a811c71d26bdeb153b3726046a6e3989369d Mon Sep 17 00:00:00 2001 From: Scott Marchant Date: Fri, 10 Jul 2026 09:25:01 -0600 Subject: [PATCH 10/10] =?UTF-8?q?feat:=20NativeConcurrency=20package=20tra?= =?UTF-8?q?it=20=E2=80=94=20NIO-free=20Swift-concurrency=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add SwiftPM traits via a versioned Package@swift-6.1.swift (base manifest restored byte-identical to the branch base for older toolchains): - traits: `NIO` (default; keeps the SwiftNIO backend exactly as today) and `NativeConcurrency` (NIO-free backend: async/await connection, [UInt8] blobs, no EventLoopFuture/ByteBuffer/NIOThreadPool). NIO products carry `.when(traits: ["NIO"])`; the KHASM_EMBEDDED env conditional is gone. - Source: the NIO-swap gates are re-keyed from `#if hasFeature(Embedded)` to `#if NativeConcurrency` (blob representation, protocol split, async connection, hooks, re-exports). Genuine Embedded language-limit gates (Encodable, LocalizedError, reflection descriptions, error interpolation, Date/Foundation) stay on `hasFeature(Embedded)`. - New on the NativeConcurrency path (it can now build on multithreaded hosts): `NativeConcurrencyLockedBox` (os_unfair_lock on Darwin, pthread_mutex elsewhere, direct access on single-threaded Embedded WASI) guarding the connection handle and row collection, and a [UInt8]-based `Data: SQLiteDataConvertible` when Foundation is present without NIO. - SQLiteConnection+WASI.swift renamed to SQLiteConnection+NativeConcurrency.swift. - VendorSQLite plugin: `nonisolated(unsafe)` on its verbose flag (plugins compile in Swift 6 mode under the tools-6.1 manifest). Verified: default `swift build` + `swift test` (55/55) unchanged; `swift build --traits NativeConcurrency` green on native macOS with zero NIO symbols in the built objects. Co-Authored-By: Claude Fable 5 --- Package.swift | 28 +--- Package@swift-6.1.swift | 135 ++++++++++++++++++ Plugins/VendorSQLite/VendorSQLite3.swift | 4 +- Sources/SQLiteNIO/Exports.swift | 8 +- Sources/SQLiteNIO/NativeConcurrencyLock.swift | 74 ++++++++++ .../SQLiteNIO/SQLiteConnection+Hooks.swift | 4 +- ... SQLiteConnection+NativeConcurrency.swift} | 28 ++-- Sources/SQLiteNIO/SQLiteConnection.swift | 4 +- Sources/SQLiteNIO/SQLiteCustomFunction.swift | 2 +- Sources/SQLiteNIO/SQLiteData.swift | 19 +-- Sources/SQLiteNIO/SQLiteDataConvertible.swift | 28 +++- Sources/SQLiteNIO/SQLiteDatabase.swift | 16 +-- Sources/SQLiteNIO/SQLiteStatement.swift | 6 +- 13 files changed, 286 insertions(+), 70 deletions(-) create mode 100644 Package@swift-6.1.swift create mode 100644 Sources/SQLiteNIO/NativeConcurrencyLock.swift rename Sources/SQLiteNIO/{SQLiteConnection+WASI.swift => SQLiteConnection+NativeConcurrency.swift} (85%) diff --git a/Package.swift b/Package.swift index d23ae6b..e448c21 100644 --- a/Package.swift +++ b/Package.swift @@ -1,5 +1,4 @@ // swift-tools-version:5.10 -import class Foundation.ProcessInfo import PackageDescription /// This list matches the [supported platforms on the Swift 5.10 release of SPM](https://github.com/swiftlang/swift-package-manager/blob/release/5.10/Sources/PackageDescription/SupportedPlatforms.swift#L34-L71) @@ -8,15 +7,6 @@ let allPlatforms: [Platform] = [.macOS, .macCatalyst, .iOS, .tvOS, .watchOS, .vi let nonWASIPlatforms: [Platform] = allPlatforms.filter { $0 != .wasi } let wasiPlatform: [Platform] = [.wasi] -// Embedded-wasm port (see /Users/scottm/git/c34/khasm/EMBEDDED_PORT_PLAN.md): with -// KHASM_EMBEDDED=1 the SwiftNIO stack is dropped entirely — the Embedded build uses the -// NIO-free, Swift-Concurrency driver over CSQLite gated in source with -// `#if hasFeature(Embedded)`. `hasFeature` is not available in manifests (they run -// host-side), hence the env-var conditional, matching khasm's own manifest gating. -// Regular (non-embedded) builds — including regular WASI — keep the NIO stack exactly -// as on the feat/khasmPAL-2026 base. -let khasmEmbedded = ProcessInfo.processInfo.environment["KHASM_EMBEDDED"] == "1" - let package = Package( name: "sqlite-nio", platforms: [ @@ -29,15 +19,10 @@ let package = Package( .library(name: "SQLiteNIO", targets: ["SQLiteNIO"]), ], dependencies: [ - // swift-nio stays on the PassiveLogic fork URL (feat/khasmPAL-2026, as on the base - // branch) rather than the local ../swift-nio clone: that clone carries the vestigial - // Option-1/2 embedded-NIO commits whose os(WASI) gates would change regular-WASI NIO - // behavior, and transitive *path* deps override URL declarations by identity in a - // consuming root graph (khasm). Under KHASM_EMBEDDED=1 NIO products are dropped, so - // the embedded build never compiles NIO from either source. + // TODO: SM: Update swift-nio version once NIOAsyncRuntime is available from swift-nio + // .package(url: "https://github.com/apple/swift-nio.git", from: "2.89.0"), .package(url: "https://github.com/PassiveLogic/swift-nio.git", branch: "feat/khasmPAL-2026"), - // Local embedded-ported clone (see /Users/scottm/git/c34/EMBEDDED_WASM_NOTES.md). - .package(path: "../swift-log"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), ], targets: [ .plugin( @@ -60,16 +45,11 @@ let package = Package( dependencies: [ .target(name: "CSQLite"), .product(name: "Logging", package: "swift-log"), - ] + (khasmEmbedded ? [] : [ - // The SwiftNIO stack is dropped on the Embedded build (KHASM_EMBEDDED=1, see the - // note at the top); the Embedded path is a NIO-free, Swift-Concurrency driver over - // CSQLite, gated in source with `#if hasFeature(Embedded)`. All regular builds - // (native + regular WASI via NIOAsyncRuntime) keep the NIO stack. .product(name: "NIOCore", package: "swift-nio"), .product(name: "NIOAsyncRuntime", package: "swift-nio", condition: .when(platforms: wasiPlatform)), .product(name: "NIOPosix", package: "swift-nio"), .product(name: "NIOFoundationCompat", package: "swift-nio"), - ]), + ], swiftSettings: swiftSettings ), .testTarget( diff --git a/Package@swift-6.1.swift b/Package@swift-6.1.swift new file mode 100644 index 0000000..e588eed --- /dev/null +++ b/Package@swift-6.1.swift @@ -0,0 +1,135 @@ +// swift-tools-version:6.1 +import PackageDescription + +/// This list matches the [supported platforms on the Swift 5.10 release of SPM](https://github.com/swiftlang/swift-package-manager/blob/release/5.10/Sources/PackageDescription/SupportedPlatforms.swift#L34-L71) +/// Don't add new platforms here unless raising the swift-tools-version of this manifest. +let allPlatforms: [Platform] = [.macOS, .macCatalyst, .iOS, .tvOS, .watchOS, .visionOS, .driverKit, .linux, .windows, .android, .wasi, .openbsd] +let nonWASIPlatforms: [Platform] = allPlatforms.filter { $0 != .wasi } +let wasiPlatform: [Platform] = [.wasi] + +let package = Package( + name: "sqlite-nio", + platforms: [ + .macOS(.v10_15), + .iOS(.v13), + .watchOS(.v6), + .tvOS(.v13), + ], + products: [ + .library(name: "SQLiteNIO", targets: ["SQLiteNIO"]), + ], + traits: [ + .default(enabledTraits: ["NIO"]), + .trait( + name: "NIO", + description: "Default backend: SwiftNIO (EventLoopFuture/ByteBuffer, NIOThreadPool)." + ), + .trait( + name: "NativeConcurrency", + description: "NIO-free backend on Swift concurrency (async/await, [UInt8] blobs). Build with `--traits NativeConcurrency` (replaces the default NIO backend)." + ), + .trait( + name: "Freestanding", + description: "Embedded/freestanding flavor (implies NativeConcurrency). No additional source effect in this package beyond NativeConcurrency; declared so a root's `--traits Freestanding` configuration names a known trait when this package is wired by path.", + enabledTraits: ["NativeConcurrency"] + ), + ], + dependencies: [ + // TODO: SM: Update swift-nio version once NIOAsyncRuntime is available from swift-nio + // .package(url: "https://github.com/apple/swift-nio.git", from: "2.89.0"), + .package(url: "https://github.com/PassiveLogic/swift-nio.git", branch: "feat/khasmPAL-2026"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), + ], + targets: [ + .plugin( + name: "VendorSQLite", + capability: .command( + intent: .custom(verb: "vendor-sqlite", description: "Vendor SQLite"), + permissions: [ + .allowNetworkConnections(scope: .all(ports: [443]), reason: "Retrieve the latest build of SQLite"), + .writeToPackageDirectory(reason: "Update the vendored SQLite files"), + ] + ), + exclude: ["001-warnings-and-data-race.patch"] + ), + .target( + name: "CSQLite", + cSettings: sqliteCSettings + ), + .target( + name: "SQLiteNIO", + dependencies: [ + .target(name: "CSQLite"), + .product(name: "Logging", package: "swift-log"), + // The SwiftNIO stack rides the default `NIO` trait. With `NativeConcurrency` + // enabled instead, SQLiteNIO is a NIO-free, Swift-Concurrency driver over + // CSQLite, gated in source with `#if NativeConcurrency`. + .product(name: "NIOCore", package: "swift-nio", condition: .when(traits: ["NIO"])), + .product(name: "NIOAsyncRuntime", package: "swift-nio", condition: .when(platforms: wasiPlatform, traits: ["NIO"])), + .product(name: "NIOPosix", package: "swift-nio", condition: .when(traits: ["NIO"])), + .product(name: "NIOFoundationCompat", package: "swift-nio", condition: .when(traits: ["NIO"])), + ], + swiftSettings: swiftSettings + ), + .testTarget( + name: "SQLiteNIOTests", + dependencies: [ + .target(name: "SQLiteNIO"), + ], + swiftSettings: swiftSettings + ), + ], + swiftLanguageModes: [.v5] +) + +var swiftSettings: [SwiftSetting] { [ + // This manifest raises the tools-version to 6.1 (for package traits); the package sources + // stay in the Swift 5 language mode of the base manifest. + .swiftLanguageMode(.v5), + .enableUpcomingFeature("ExistentialAny"), + .enableUpcomingFeature("ConciseMagicFile"), + .enableUpcomingFeature("ForwardTrailingClosures"), + .enableUpcomingFeature("DisableOutwardActorInference"), + .enableExperimentalFeature("StrictConcurrency=complete"), +] } + +var sqliteCSettings: [CSetting] { [ + // Derived from sqlite3 version 3.43.0 + .define("SQLITE_DEFAULT_MEMSTATUS", to: "0"), + .define("SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS"), + .define("SQLITE_DQS", to: "0"), + .define("SQLITE_ENABLE_API_ARMOR", .when(configuration: .debug)), + .define("SQLITE_ENABLE_COLUMN_METADATA"), + .define("SQLITE_ENABLE_DBSTAT_VTAB"), + .define("SQLITE_ENABLE_FTS3"), + .define("SQLITE_ENABLE_FTS3_PARENTHESIS"), + .define("SQLITE_ENABLE_FTS3_TOKENIZER"), + .define("SQLITE_ENABLE_FTS4"), + .define("SQLITE_ENABLE_FTS5"), + .define("SQLITE_ENABLE_NULL_TRIM"), + .define("SQLITE_ENABLE_RTREE"), + .define("SQLITE_ENABLE_SESSION"), + .define("SQLITE_ENABLE_STMTVTAB"), + .define("SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION"), + .define("SQLITE_ENABLE_UNLOCK_NOTIFY"), + .define("SQLITE_MAX_VARIABLE_NUMBER", to: "250000"), + .define("SQLITE_LIKE_DOESNT_MATCH_BLOBS"), + .define("SQLITE_OMIT_COMPLETE"), + .define("SQLITE_OMIT_DEPRECATED"), + .define("SQLITE_OMIT_DESERIALIZE"), + .define("SQLITE_OMIT_GET_TABLE"), + .define("SQLITE_OMIT_LOAD_EXTENSION"), + .define("SQLITE_OMIT_PROGRESS_CALLBACK"), + .define("SQLITE_OMIT_SHARED_CACHE"), + .define("SQLITE_OMIT_TCL_VARIABLE"), + .define("SQLITE_OMIT_TRACE"), + .define("SQLITE_SECURE_DELETE"), + .define("SQLITE_THREADSAFE", to: "1", .when(platforms: nonWASIPlatforms)), + // For now, we use the single threaded sqlite variation for the WASI platform + // since single-threaded operation is the least common denominator capability + // for Wasm executables and it is considered unreliable to use canImport(wasi_pthread) + // in a manifest file to distinguish between the two capabilities. + .define("SQLITE_THREADSAFE", to: "0", .when(platforms: wasiPlatform)), + .define("SQLITE_UNTESTABLE"), + .define("SQLITE_USE_URI"), +] } diff --git a/Plugins/VendorSQLite/VendorSQLite3.swift b/Plugins/VendorSQLite/VendorSQLite3.swift index 5d5da7c..8fc6445 100644 --- a/Plugins/VendorSQLite/VendorSQLite3.swift +++ b/Plugins/VendorSQLite/VendorSQLite3.swift @@ -19,7 +19,9 @@ struct VendorSQLite: CommandPlugin { static let sqliteURL = URL(string: "https://sqlite.org")! static let vendorPrefix = "sqlite_nio" - static var verbose = false + // `nonisolated(unsafe)`: set once from `performCommand()` before any concurrent work; needed + // because plugins compile in the Swift 6 language mode under the tools-6.1 traits manifest. + nonisolated(unsafe) static var verbose = false var verbose: Bool { Self.verbose } diff --git a/Sources/SQLiteNIO/Exports.swift b/Sources/SQLiteNIO/Exports.swift index 9f144f3..e1a334f 100644 --- a/Sources/SQLiteNIO/Exports.swift +++ b/Sources/SQLiteNIO/Exports.swift @@ -1,6 +1,6 @@ -// These re-exports are all SwiftNIO types, which are elided on the Embedded build (the NIO-free -// Embedded path uses Swift Concurrency + `[UInt8]` instead of EventLoop/ByteBuffer). -#if !hasFeature(Embedded) +// These re-exports are all SwiftNIO types, which are elided on the NativeConcurrency build (the +// NIO-free path uses Swift Concurrency + `[UInt8]` instead of EventLoop/ByteBuffer). +#if !NativeConcurrency @_documentation(visibility: internal) @_exported import struct NIOCore.ByteBuffer // See SQLiteConnection.swift for why we gate on `os(WASI)` rather than @@ -22,4 +22,4 @@ #else @_documentation(visibility: internal) @_exported import class NIOPosix.MultiThreadedEventLoopGroup #endif -#endif // !hasFeature(Embedded) +#endif // !NativeConcurrency diff --git a/Sources/SQLiteNIO/NativeConcurrencyLock.swift b/Sources/SQLiteNIO/NativeConcurrencyLock.swift new file mode 100644 index 0000000..6053d7d --- /dev/null +++ b/Sources/SQLiteNIO/NativeConcurrencyLock.swift @@ -0,0 +1,74 @@ +#if NativeConcurrency +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#elseif canImport(Android) +import Android +#endif + +/// A minimal mutex-protected value box for the NativeConcurrency (NIO-free) build. +/// +/// `NIOConcurrencyHelpers.NIOLock` is unavailable here (NIO is not linked), and +/// `Synchronization.Mutex` would force a platform-floor bump (macOS 15 et al.), so this uses +/// `os_unfair_lock` on Darwin and `pthread_mutex_t` elsewhere. On single-threaded targets with +/// no lock primitive (Embedded WASI) it degrades to direct access, which is sound because that +/// runtime has exactly one thread. +final class NativeConcurrencyLockedBox: @unchecked Sendable { + #if hasFeature(Embedded) || os(WASI) + private var value: Value + + init(_ value: Value) { + self.value = value + } + + func withLock(_ body: (inout Value) throws -> Result) rethrows -> Result { + try body(&self.value) + } + #elseif canImport(Darwin) + private let lockPointer: os_unfair_lock_t + private var value: Value + + init(_ value: Value) { + self.lockPointer = .allocate(capacity: 1) + self.lockPointer.initialize(to: os_unfair_lock()) + self.value = value + } + + deinit { + self.lockPointer.deinitialize(count: 1) + self.lockPointer.deallocate() + } + + func withLock(_ body: (inout Value) throws -> Result) rethrows -> Result { + os_unfair_lock_lock(self.lockPointer) + defer { os_unfair_lock_unlock(self.lockPointer) } + return try body(&self.value) + } + #else + private let mutexPointer: UnsafeMutablePointer + private var value: Value + + init(_ value: Value) { + self.mutexPointer = .allocate(capacity: 1) + self.mutexPointer.initialize(to: pthread_mutex_t()) + pthread_mutex_init(self.mutexPointer, nil) + self.value = value + } + + deinit { + pthread_mutex_destroy(self.mutexPointer) + self.mutexPointer.deinitialize(count: 1) + self.mutexPointer.deallocate() + } + + func withLock(_ body: (inout Value) throws -> Result) rethrows -> Result { + pthread_mutex_lock(self.mutexPointer) + defer { pthread_mutex_unlock(self.mutexPointer) } + return try body(&self.value) + } + #endif +} +#endif // NativeConcurrency diff --git a/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift b/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift index ab955eb..dff89e5 100644 --- a/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift +++ b/Sources/SQLiteNIO/SQLiteConnection+Hooks.swift @@ -1,4 +1,4 @@ -#if !hasFeature(Embedded) // EMBEDDED-WASI: NIO-based connection; the WASI build uses SQLiteConnection+WASI.swift +#if !NativeConcurrency // NIO-based hooks; the NativeConcurrency build uses SQLiteConnection+NativeConcurrency.swift (no hooks surface) import Foundation import NIOConcurrencyHelpers import NIOCore @@ -867,4 +867,4 @@ extension SQLiteConnection { } } -#endif // !hasFeature(Embedded) +#endif // !NativeConcurrency diff --git a/Sources/SQLiteNIO/SQLiteConnection+WASI.swift b/Sources/SQLiteNIO/SQLiteConnection+NativeConcurrency.swift similarity index 85% rename from Sources/SQLiteNIO/SQLiteConnection+WASI.swift rename to Sources/SQLiteNIO/SQLiteConnection+NativeConcurrency.swift index 6b30a00..201f9fc 100644 --- a/Sources/SQLiteNIO/SQLiteConnection+WASI.swift +++ b/Sources/SQLiteNIO/SQLiteConnection+NativeConcurrency.swift @@ -1,26 +1,32 @@ -#if hasFeature(Embedded) +#if NativeConcurrency import CSQLite import Logging /// A wrapper for the `OpaquePointer` used to represent an open `sqlite3` handle. /// -/// On WASI the runtime is single-threaded, so no thread pool / locking is involved; the -/// `@unchecked Sendable` is justified the same way as the SwiftNIO build (serialized SQLite, -/// `SQLITE_OPEN_FULLMUTEX`), and the handle is only mutated to `nil` on close. +/// The pointer itself is guarded by a lock (it is read on every query and written to `nil` on +/// close); the SQLite handle behind it is opened `SQLITE_OPEN_FULLMUTEX`, so concurrent use of +/// the handle is serialized by SQLite itself, as on the SwiftNIO build. On single-threaded +/// targets (Embedded WASI) the lock degrades to direct access. final class SQLiteConnectionHandle: @unchecked Sendable { - var raw: OpaquePointer? + private let storage: NativeConcurrencyLockedBox + + var raw: OpaquePointer? { + get { self.storage.withLock { $0 } } + set { self.storage.withLock { $0 = newValue } } + } init(_ raw: OpaquePointer?) { - self.raw = raw + self.storage = .init(raw) } } -/// A single open connection to an SQLite database (WASI / Embedded build). +/// A single open connection to an SQLite database (NativeConcurrency build). /// /// This is the NIO-free variant: it exposes a Swift-Concurrency (`async`/`await`) API over CSQLite -/// with no `EventLoopFuture`, `EventLoopGroup`, or `NIOThreadPool`. Because WebAssembly is -/// single-threaded, the blocking libsqlite3 calls run inline on the calling task. The observable -/// hook API and connection pooling are not available on this build. +/// with no `EventLoopFuture`, `EventLoopGroup`, or `NIOThreadPool`. The blocking libsqlite3 calls +/// run inline on the calling task. The observable hook API and connection pooling are not +/// available on this build. public final class SQLiteConnection: SQLiteDatabase, Sendable { /// The possible storage types for an SQLite database. public enum Storage: Equatable, Sendable { @@ -145,4 +151,4 @@ public final class SQLiteConnection: SQLiteDatabase, Sendable { assert(self.handle.raw == nil, "SQLiteConnection was not closed before deinitializing") } } -#endif // hasFeature(Embedded) +#endif // NativeConcurrency diff --git a/Sources/SQLiteNIO/SQLiteConnection.swift b/Sources/SQLiteNIO/SQLiteConnection.swift index 5de3bbf..572ce29 100644 --- a/Sources/SQLiteNIO/SQLiteConnection.swift +++ b/Sources/SQLiteNIO/SQLiteConnection.swift @@ -1,4 +1,4 @@ -#if !hasFeature(Embedded) // EMBEDDED: NIO-based connection; the Embedded build uses SQLiteConnection+WASI.swift +#if !NativeConcurrency // NIO-based connection; the NativeConcurrency build uses SQLiteConnection+NativeConcurrency.swift import NIOConcurrencyHelpers import NIOCore // Use NIOPosix on every host platform that supports it (macOS, Linux, etc.) @@ -459,4 +459,4 @@ extension SQLiteConnection { } } -#endif // !hasFeature(Embedded) +#endif // !NativeConcurrency diff --git a/Sources/SQLiteNIO/SQLiteCustomFunction.swift b/Sources/SQLiteNIO/SQLiteCustomFunction.swift index ada926f..935ee01 100644 --- a/Sources/SQLiteNIO/SQLiteCustomFunction.swift +++ b/Sources/SQLiteNIO/SQLiteCustomFunction.swift @@ -300,7 +300,7 @@ public final class SQLiteCustomFunction: Hashable { case .text(let string): sqlite_nio_sqlite3_result_text(sqliteContext, string, -1, SQLITE_TRANSIENT) case .blob(let value): - #if hasFeature(Embedded) + #if NativeConcurrency value.withUnsafeBytes { pointer in sqlite_nio_sqlite3_result_blob(sqliteContext, pointer.baseAddress, Int32(value.count), SQLITE_TRANSIENT) } diff --git a/Sources/SQLiteNIO/SQLiteData.swift b/Sources/SQLiteNIO/SQLiteData.swift index ba5dd6a..850ef65 100644 --- a/Sources/SQLiteNIO/SQLiteData.swift +++ b/Sources/SQLiteNIO/SQLiteData.swift @@ -1,5 +1,5 @@ import CSQLite -#if !hasFeature(Embedded) +#if !NativeConcurrency import NIOCore #endif @@ -31,8 +31,9 @@ public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { /// `TEXT` affinity, represented in Swift by `String`. case text(String) - /// `BLOB` affinity. Represented by SwiftNIO's `ByteBuffer`, or by `[UInt8]` on WASI (NIO-free). - #if hasFeature(Embedded) + /// `BLOB` affinity. Represented by SwiftNIO's `ByteBuffer`, or by `[UInt8]` on the + /// NativeConcurrency (NIO-free) build. + #if NativeConcurrency case blob([UInt8]) #else case blob(ByteBuffer) @@ -103,7 +104,7 @@ public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { /// Returns the data as a blob, if it has `BLOB` affinity. /// /// `INTEGER`, `REAL`, `TEXT`, and `NULL` values always return `nil`. - #if hasFeature(Embedded) + #if NativeConcurrency public var blob: [UInt8]? { switch self { case .blob(let buffer): return buffer @@ -134,7 +135,7 @@ public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { // See `CustomStringConvertible.description`. public var description: String { switch self { - #if hasFeature(Embedded) + #if NativeConcurrency case .blob(let data): return "<\(data.count) bytes>" #else case .blob(let data): return "<\(data.readableBytes) bytes>" @@ -154,8 +155,8 @@ public enum SQLiteData: Equatable, CustomStringConvertible, Sendable { case .integer(let value): try container.encode(value) case .float(let value): try container.encode(value) case .text(let value): try container.encode(value) - #if hasFeature(Embedded) - case .blob(let value): try container.encode(value) // [UInt8] encodes as raw bytes + #if NativeConcurrency + case .blob(let value): try container.encode(value) // [UInt8] encodes as raw bytes, matching the ByteBuffer branch #else case .blob(let value): try container.encode(Array(value.readableBytesView)) // N.B.: Don't use ByteBuffer's Codable conformance; it encodes as Base64, not raw bytes #endif @@ -188,14 +189,14 @@ extension SQLiteData { case SQLITE_BLOB: if let bytes = sqlite_nio_sqlite3_value_blob(sqliteValue) { let count = Int(sqlite_nio_sqlite3_value_bytes(sqliteValue)) - #if hasFeature(Embedded) + #if NativeConcurrency self = .blob([UInt8](UnsafeRawBufferPointer(start: bytes, count: count))) // copy bytes #else let buffer = ByteBuffer(bytes: UnsafeRawBufferPointer(start: bytes, count: count)) self = .blob(buffer) // copy bytes #endif } else { - #if hasFeature(Embedded) + #if NativeConcurrency self = .blob([]) #else self = .blob(ByteBuffer()) diff --git a/Sources/SQLiteNIO/SQLiteDataConvertible.swift b/Sources/SQLiteNIO/SQLiteDataConvertible.swift index 3e1fd1e..b97e734 100644 --- a/Sources/SQLiteNIO/SQLiteDataConvertible.swift +++ b/Sources/SQLiteNIO/SQLiteDataConvertible.swift @@ -1,8 +1,10 @@ -// ByteBuffer (NIOCore), Data/Date (Foundation), and the NIOFoundationCompat bridges are all elided -// on WASI; see the `#if hasFeature(Embedded)` / `#if !hasFeature(Embedded)` conformances below. -#if !hasFeature(Embedded) +// ByteBuffer (NIOCore) and the NIOFoundationCompat bridges are elided on the NativeConcurrency +// (NIO-free) build; Foundation (`Data`/`Date`) is used whenever it is available. +#if !NativeConcurrency import NIOCore import NIOFoundationCompat +#endif +#if canImport(Foundation) import Foundation #endif @@ -78,7 +80,7 @@ extension Float: SQLiteDataConvertible { } } -#if hasFeature(Embedded) +#if NativeConcurrency extension [UInt8]: SQLiteDataConvertible { public init?(sqliteData: SQLiteData) { guard case .blob(let value) = sqliteData else { @@ -91,6 +93,21 @@ extension [UInt8]: SQLiteDataConvertible { .blob(self) } } + +#if canImport(Foundation) +extension Data: SQLiteDataConvertible { + public init?(sqliteData: SQLiteData) { + guard case .blob(let value) = sqliteData else { + return nil + } + self = .init(value) + } + + public var sqliteData: SQLiteData? { + .blob([UInt8](self)) + } +} +#endif #else extension ByteBuffer: SQLiteDataConvertible { public init?(sqliteData: SQLiteData) { @@ -132,7 +149,8 @@ extension Bool: SQLiteDataConvertible { } } -// Date conversions rely on Foundation (`Date`, `ISO8601DateFormatter`), unavailable on WASI. +// Date conversions rely on Foundation (`Date`, `ISO8601DateFormatter`), unavailable in +// Embedded Swift. #if !hasFeature(Embedded) extension Date: SQLiteDataConvertible { public init?(sqliteData: SQLiteData) { diff --git a/Sources/SQLiteNIO/SQLiteDatabase.swift b/Sources/SQLiteNIO/SQLiteDatabase.swift index 12bb2b7..29e69d2 100644 --- a/Sources/SQLiteNIO/SQLiteDatabase.swift +++ b/Sources/SQLiteNIO/SQLiteDatabase.swift @@ -1,4 +1,4 @@ -#if !hasFeature(Embedded) // EMBEDDED-WASI: NIO/EventLoopFuture protocol; WASI uses the async protocol below +#if !NativeConcurrency // NIO/EventLoopFuture protocol; the NativeConcurrency build uses the async protocol below import NIOCore import CSQLite import Logging @@ -197,13 +197,13 @@ private struct SQLiteDatabaseCustomLogger: SQLiteDatabase { } } -#endif // !hasFeature(Embedded) +#endif // !NativeConcurrency -#if hasFeature(Embedded) +#if NativeConcurrency import CSQLite import Logging -/// NIO-free (`async`/`await`) variant of ``SQLiteDatabase`` for the WASI / Embedded build. +/// NIO-free (`async`/`await`) variant of ``SQLiteDatabase`` for the NativeConcurrency build. /// /// The protocol deliberately has only non-generic requirements so it remains usable as an /// existential (`any SQLiteDatabase`) under Embedded Swift, which cannot place a generic method in a @@ -234,9 +234,9 @@ extension SQLiteDatabase { /// Execute a query and collect the result rows. public func query(_ query: String, _ binds: [SQLiteData] = []) async throws -> [SQLiteRow] { - nonisolated(unsafe) var rows: [SQLiteRow] = [] - try await self.query(query, binds) { rows.append($0) } - return rows + let rows = NativeConcurrencyLockedBox<[SQLiteRow]>([]) + try await self.query(query, binds) { row in rows.withLock { $0.append(row) } } + return rows.withLock { $0 } } /// Return a database that logs to `logger`, forwarding everything else to `self`. @@ -263,4 +263,4 @@ private struct SQLiteDatabaseCustomLogger: SQLiteDatabase { Self(database: self.database, logger: logger) } } -#endif // hasFeature(Embedded) +#endif // NativeConcurrency diff --git a/Sources/SQLiteNIO/SQLiteStatement.swift b/Sources/SQLiteNIO/SQLiteStatement.swift index ac1eece..6b71cea 100644 --- a/Sources/SQLiteNIO/SQLiteStatement.swift +++ b/Sources/SQLiteNIO/SQLiteStatement.swift @@ -1,4 +1,4 @@ -#if !hasFeature(Embedded) +#if !NativeConcurrency import NIOCore #endif import CSQLite @@ -42,7 +42,7 @@ struct SQLiteStatement { switch bind { case .blob(let value): - #if hasFeature(Embedded) + #if NativeConcurrency ret = value.withUnsafeBytes { sqlite_nio_sqlite3_bind_blob64(self.handle, i, $0.baseAddress, UInt64($0.count), SQLITE_TRANSIENT) } @@ -110,7 +110,7 @@ struct SQLiteStatement { return .text(.init(cString: val)) case SQLITE_BLOB: let length = Int(sqlite_nio_sqlite3_column_bytes(self.handle, offset)) - #if hasFeature(Embedded) + #if NativeConcurrency var bytes = [UInt8]() bytes.reserveCapacity(length) if let blobPointer = sqlite_nio_sqlite3_column_blob(self.handle, offset) {