From dd1231abe4919a5adcc2175769843943b1472675 Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Mon, 27 Jul 2026 12:10:04 -0400 Subject: [PATCH 1/9] feat(db): add replica_heartbeat table for portable fence observation Single CHECK'd-row table whose monotonic token, committed after the fence's ordered writer scan, gives readers a freshness observation that Aurora reader endpoints can actually serve (unlike pg_last_wal_replay_lsn, which they hide). The epoch column detects token resets. Registered as operator-global: it describes replication topology, not tenant data. Numbered 0026: version 25 was burned by 0025_users_agent_owner_lookup (#2615, reverted by #3168). DBs migrated during that window have version 25 recorded with those bytes; a different 0025 would fail their startup with VersionMismatch. 25 stays vacant as a tombstone. Migration shape assertions extended; the operator-global allowlist in the tenant-scoping lint gains the new table. Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- crates/buzz-db/src/migration.rs | 23 +++++++++++++++- migrations/0026_replica_heartbeat.sql | 38 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 migrations/0026_replica_heartbeat.sql diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 1674b0ec4d..ab28720f95 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -347,6 +347,7 @@ mod tests { "push_gateway_delivery_auth_replays", "push_gateway_delivery_request_replays", "product_feedback", + "replica_heartbeat", ] { if normalized[insert_pos..].contains(&format!("'{value}'")) { globals.insert(value.to_owned()); @@ -560,7 +561,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 24); + assert_eq!(migrations.len(), 25); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -879,6 +880,26 @@ mod tests { .to_lowercase() .contains("for update")); assert!(ttl_shared.contains("NEW.kind <> 9007")); + + // Replica heartbeat: the fence's portable read-side observation. A + // single CHECK'd row makes the token update the serialization point + // (multi-pod commit ordering), and the epoch column is what detects + // token resets — both are load-bearing for the routing proof. + // + // Version 26, not 25: version 25 was burned by + // `0025_users_agent_owner_lookup` (#2615), which shipped on main and + // was then reverted (#3168). Databases migrated during that window + // have version 25 recorded with those bytes; reusing the number with + // different content would fail their startup with VersionMismatch. + // 25 stays vacant as a tombstone (a re-land of #2615 must reuse the + // exact original bytes). + assert_eq!(migrations[24].version, 26); + let heartbeat = migrations[24].sql.as_str(); + assert!(heartbeat.contains("CREATE TABLE replica_heartbeat")); + assert!(heartbeat.contains("CHECK (id = 1)")); + assert!(heartbeat.contains("epoch")); + assert!(heartbeat.contains("INSERT INTO replica_heartbeat (id) VALUES (1)")); + assert!(heartbeat.contains("_operator_global_tables")); } #[test] diff --git a/migrations/0026_replica_heartbeat.sql b/migrations/0026_replica_heartbeat.sql new file mode 100644 index 0000000000..278be4d87a --- /dev/null +++ b/migrations/0026_replica_heartbeat.sql @@ -0,0 +1,38 @@ +-- Replica heartbeat: a portable read-side freshness observation for the +-- replica fence (see crates/buzz-db/src/replica_fence.rs). +-- +-- Why: the fence's ordered writer-side proof previously ended in a WAL-LSN +-- comparison (`pg_last_wal_replay_lsn() >= L`), which Aurora's reader +-- endpoints do not expose — the fence therefore never opened on Aurora, by +-- design (fail closed). This table replaces only that read-side observation: +-- the probe commits a monotonically increasing `token` AFTER the ordered +-- writer scan (clock sample -> oldest-xact guard), and a reader session that +-- observes token >= M has, by WAL/storage replay order, also replayed every +-- commit that preceded M. The commit-time floor guard (migration 0021) and +-- the writer-side scan remain load-bearing and unchanged. +-- +-- Shape: exactly one row, enforced by the CHECK'd primary key. Every relay +-- pod's probe increments the same row; the single-row UPDATE is the +-- serialization point that makes tokens globally commit-ordered, which is +-- what lets a pod prove coverage from the greatest token it retained that is +-- <= the token a reader session observes (multi-pod safety). +-- +-- `epoch` detects resets: a restore/re-seed that rolls `token` backwards +-- must never let a stale retained token masquerade as fresh coverage. +-- Readers validate the observed epoch against the epoch retained with each +-- token; a mismatch fails closed (route to writer). +-- +-- Not an events row: exempt from the created_at floor guard by construction, +-- and deliberately deployment-global (no community_id) — it describes the +-- replication topology, not tenant data. + +CREATE TABLE replica_heartbeat ( + id smallint PRIMARY KEY CHECK (id = 1), + epoch uuid NOT NULL DEFAULT gen_random_uuid(), + token bigint NOT NULL DEFAULT 0 +); + +INSERT INTO replica_heartbeat (id) VALUES (1); + +INSERT INTO _operator_global_tables (table_name, reason) VALUES + ('replica_heartbeat', 'single-row replication freshness token; describes deployment topology, never tenant data'); From 4aa7cfddbfe453a8534c6df9ddeb3b6c76ca61ea Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Mon, 27 Jul 2026 17:37:39 -0400 Subject: [PATCH 2/9] fix(db): renumber replica_heartbeat migration to 0025 The 0025 slot was briefly occupied by 0025_users_agent_owner_lookup (#2615), but it lived on main for only ~1 hour before the revert (#3168) and never reached any deployed image (prod rolled dd222a5 -> sha-137185e, both post-revert or pre-add). No production database has version 25 recorded, so the number is free; a dev DB migrated inside that window is already broken on current main (VersionMissing) and is repaired by deleting its stale version-25 row. Per Tyler in thread 39d1e174. Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- crates/buzz-db/src/migration.rs | 16 ++++++++-------- ..._heartbeat.sql => 0025_replica_heartbeat.sql} | 0 2 files changed, 8 insertions(+), 8 deletions(-) rename migrations/{0026_replica_heartbeat.sql => 0025_replica_heartbeat.sql} (100%) diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index ab28720f95..98ce170f12 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -886,14 +886,14 @@ mod tests { // (multi-pod commit ordering), and the epoch column is what detects // token resets — both are load-bearing for the routing proof. // - // Version 26, not 25: version 25 was burned by - // `0025_users_agent_owner_lookup` (#2615), which shipped on main and - // was then reverted (#3168). Databases migrated during that window - // have version 25 recorded with those bytes; reusing the number with - // different content would fail their startup with VersionMismatch. - // 25 stays vacant as a tombstone (a re-land of #2615 must reuse the - // exact original bytes). - assert_eq!(migrations[24].version, 26); + // Version 25 is safe to use despite `0025_users_agent_owner_lookup` + // (#2615) briefly occupying it: that migration lived on main for + // ~1 hour before the revert (#3168), and no deployed image ever + // contained the file (prod rolled dd222a5 → sha-137185e, which + // already includes the revert). A dev database migrated from a + // build inside that window is broken on current main regardless + // (VersionMissing); the repair is deleting its stale version-25 row. + assert_eq!(migrations[24].version, 25); let heartbeat = migrations[24].sql.as_str(); assert!(heartbeat.contains("CREATE TABLE replica_heartbeat")); assert!(heartbeat.contains("CHECK (id = 1)")); diff --git a/migrations/0026_replica_heartbeat.sql b/migrations/0025_replica_heartbeat.sql similarity index 100% rename from migrations/0026_replica_heartbeat.sql rename to migrations/0025_replica_heartbeat.sql From ca801bda9a34b5da6d4366ab6be0b9c3bc545385 Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Mon, 27 Jul 2026 22:39:07 -0400 Subject: [PATCH 3/9] feat(replica): heartbeat-token fence with connection-local reader routing Replace the WAL-LSN read-side observation (pg_last_wal_replay_lsn, which Aurora reader endpoints hide) with a portable heartbeat token, and make the freshness proof connection-local per Rev 2. Probe (replica_fence.rs): the ordered writer-side scan is unchanged and load-bearing (S = clock_timestamp -> pg_stat_activity oldest-xact scan, masked-visibility fail-closed) but now ends by committing a heartbeat token LAST on the same pinned connection (single-row UPDATE ... RETURNING token, epoch on replica_heartbeat, migration 0026). The single-row update serializes all pods' probes, so tokens are globally commit-ordered and a reader session observing token >= M has replayed everything the three-bucket argument covers for M. Each probe retains (token, committed_at, fence_wall) in a bounded ring; epoch changes reset the ring, and a same-epoch token regression (restore adversary) clears it and rotates the epoch on the writer so pre-rewind readers fail the epoch check instead of proving stale coverage. Routing (lib.rs): eligibility is decided per request ON the serving connection. A routed read acquires a reader session, observes token/epoch there (observe_heartbeat, with backend identity for live evidence), resolves the strongest retained entry <= the observation, and serves the page on that same session. Cursor pages keep today's completeness math (Predicate B: cursor_ts <= proved fence_wall; thread pages keep the full-page + tail-below-wall writer re-verification). Head fetches are Predicate A (bounded staleness): routed only when BUZZ_REPLICA_HEAD_MAX_AGE_SECS is set (> 0; default off, clamped to the fence staleness gate) and the proved entry is within budget. Everything fails closed to the writer: acquire error, missing heartbeat row, epoch mismatch, token below the ring, over-budget entry. Aux closure (bridge.rs): get_channel_window_with_session returns the serving session as a ReadSession, and the window aux closure runs its hops on that exact connection - hopping pooled reader sessions would discard the proof. Observability: buzz_db_route_decision{path, decision, reason} counters (channel_head/channel_cursor/thread_head/thread_cursor/thread_eof x replica/writer x fresh/covered/stale/reader_token_behind/ reader_validation_error/uninitialized/disabled), a buzz_db_replica_heartbeat_age_seconds gauge, and debug logs with the proved token and backend identity. Probe cadence moves 5s -> 1s (head budget needs cadence well under B; cost is one single-row UPDATE tuple of WAL per beat per pod). Tests: resolve/record/ring/rotation unit coverage; probe + rotation against private scratch databases; head-gate routing (off by default, replica within budget, writer when over budget) and the existing divergent-fixture routing suite updated to the connection-local flow; config parse coverage for BUZZ_REPLICA_HEAD_MAX_AGE_SECS. Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- Cargo.lock | 1 + crates/buzz-db/Cargo.toml | 1 + crates/buzz-db/src/event.rs | 13 +- crates/buzz-db/src/lib.rs | 515 ++++++++++++++++++-- crates/buzz-db/src/migration.rs | 2 +- crates/buzz-db/src/replica_fence.rs | 731 ++++++++++++++++++++++------ crates/buzz-db/src/thread.rs | 55 ++- crates/buzz-relay/src/api/bridge.rs | 13 +- crates/buzz-relay/src/config.rs | 54 ++ crates/buzz-relay/src/main.rs | 7 + 10 files changed, 1190 insertions(+), 202 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b60dc4579..2fc3158a02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -949,6 +949,7 @@ dependencies = [ "buzz-core", "chrono", "hex", + "metrics", "nostr", "rand 0.10.1", "serde", diff --git a/crates/buzz-db/Cargo.toml b/crates/buzz-db/Cargo.toml index 01f1e172b6..dc2304368e 100644 --- a/crates/buzz-db/Cargo.toml +++ b/crates/buzz-db/Cargo.toml @@ -21,6 +21,7 @@ tracing = { workspace = true } thiserror = { workspace = true } nostr = { workspace = true } rand = { workspace = true } +metrics = { workspace = true } [dev-dependencies] tokio = { workspace = true } diff --git a/crates/buzz-db/src/event.rs b/crates/buzz-db/src/event.rs index 520fd1536f..70eaa5c28a 100644 --- a/crates/buzz-db/src/event.rs +++ b/crates/buzz-db/src/event.rs @@ -316,6 +316,17 @@ pub async fn insert_event( /// Uses `QueryBuilder` for dynamic filter composition — avoids string concatenation /// while keeping all user values in bind parameters. pub async fn query_events(pool: &PgPool, q: &EventQuery) -> Result> { + let mut conn = pool.acquire().await?; + query_events_on(&mut conn, q).await +} + +/// [`query_events`] on a specific session — the replica-routing path runs +/// follow-up (aux) queries on the exact reader connection whose heartbeat +/// observation proved coverage for the page they annotate. +pub(crate) async fn query_events_on( + conn: &mut sqlx::PgConnection, + q: &EventQuery, +) -> Result> { // Composite cursor requires both halves. if q.before_id.is_some() && q.until.is_none() { return Err(DbError::InvalidData( @@ -538,7 +549,7 @@ pub async fn query_events(pool: &PgPool, q: &EventQuery) -> Result, + /// Head-fetch routing budget (Predicate A): a head page may be served + /// from a proved replica session only when the proved heartbeat entry is + /// at most this old. `None` disables head routing entirely (the rollout + /// default) — bounded-stale head semantics are a product decision, not + /// an invariant, so the gate ships off. + pub(crate) replica_head_max_age: Option, +} + +/// The session that served (or will serve) a routed read, so follow-up +/// queries in the same request (the channel-window aux closure) run on the +/// **same proved connection** — a different pooled reader session may sit at +/// a different replay position, so hopping sessions would discard the proof. +/// +/// `Writer` carries the writer pool: follow-ups there are authoritative by +/// construction and need no session pinning. +pub struct ReadSession { + inner: ReadSessionInner, +} + +enum ReadSessionInner { + /// A replica session whose heartbeat observation proved fence coverage. + Replica(sqlx::pool::PoolConnection), + /// The writer pool (cheap clone; Arc-backed). + Writer(PgPool), +} + +impl ReadSession { + /// Query events on this session (see [`Db::query_events`]). + pub async fn query_events(&mut self, q: &EventQuery) -> Result> { + match &mut self.inner { + ReadSessionInner::Replica(conn) => event::query_events_on(conn, q).await, + ReadSessionInner::Writer(pool) => event::query_events(pool, q).await, + } + } + + /// Whether this session is a proved replica connection (observability). + pub fn is_replica(&self) -> bool { + matches!(self.inner, ReadSessionInner::Replica(_)) + } +} + +/// Where one routed read is served (see [`Db::route_read`]). +enum RouteDecision { + /// A reader session that proved this fence entry — the page runs here. + Replica( + sqlx::pool::PoolConnection, + replica_fence::TokenEntry, + ), + /// Fail closed: serve from the writer pool. + Writer, +} + +/// The predicate one routed read must satisfy (see [`Db::route_read`]). +enum RoutePredicate { + /// Predicate A — head fetch, bounded staleness: the proved entry must + /// be within the configured head budget (default off). + Head, + /// Predicate B — cursor page, completeness: the proved wall must cover + /// the cursor timestamp. `None` means the caller could not derive an + /// upper bound from its cursor (forward-walking thread pages) and + /// post-verifies the served rows against the proved wall itself. + Cursor(Option>), +} + +impl RoutePredicate { + /// A channel-window request: cursor pages are bounded above by the + /// cursor timestamp (Predicate B); a head fetch is Predicate A. + fn from_channel_cursor(cursor: &Option<(DateTime, Vec)>) -> Self { + match cursor { + Some((ts, _)) => RoutePredicate::Cursor(Some(*ts)), + None => RoutePredicate::Head, + } + } +} + +/// Map the configured head budget (`BUZZ_REPLICA_HEAD_MAX_AGE_SECS`) to the +/// runtime gate: `0` disables head routing; anything above the fence +/// staleness gate is clamped to it (an entry older than the staleness gate +/// never routes anyway, so a larger budget would only misrepresent the +/// config). +fn head_budget_from_secs(secs: u64) -> Option { + match secs { + 0 => None, + secs => Some(Duration::from_secs(secs).min(replica_fence::FENCE_STALENESS)), + } } /// Snapshot of Postgres connection pool utilisation. @@ -240,6 +326,12 @@ pub struct DbConfig { pub max_lifetime_secs: u64, /// Seconds a connection may sit idle before being closed. pub idle_timeout_secs: u64, + /// Head-fetch replica budget `B` in seconds (Predicate A, env + /// `BUZZ_REPLICA_HEAD_MAX_AGE_SECS`). `0` disables head routing — the + /// rollout default. Values above [`replica_fence::FENCE_STALENESS`] are + /// clamped to it: an entry older than the staleness gate never routes + /// anyway, so a larger budget would only misrepresent the config. + pub replica_head_max_age_secs: u64, } impl Default for DbConfig { @@ -255,6 +347,7 @@ impl Default for DbConfig { acquire_timeout_secs: 3, max_lifetime_secs: 1800, idle_timeout_secs: 600, + replica_head_max_age_secs: 0, } } } @@ -365,11 +458,13 @@ impl Db { Some(url) => Some(Self::connect_pool(config, url, false).await?), None => None, }; + let replica_head_max_age = head_budget_from_secs(config.replica_head_max_age_secs); Ok(Self { pool, max_connections: config.max_connections, read_pool, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), + replica_head_max_age, }) } @@ -408,6 +503,7 @@ impl Db { pool, read_pool: None, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), + replica_head_max_age: None, } } @@ -424,9 +520,16 @@ impl Db { pool, read_pool: Some(read_pool), fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), + replica_head_max_age: None, } } + /// Test hook: set the head-fetch routing budget (Predicate A), which + /// [`Db::from_pools`] leaves disabled. + pub fn set_replica_head_max_age_for_tests(&mut self, budget: Option) { + self.replica_head_max_age = budget; + } + /// The freshness fence gating replica routing (see [`replica_fence`]). pub fn fence(&self) -> &std::sync::Arc { &self.fence @@ -438,7 +541,7 @@ impl Db { /// Ordering matters (Perci, PR #2084 review): this must run **after** /// the migration decision. On a relay with `BUZZ_AUTO_MIGRATE` off, the /// writer pool arms the GUC regardless, but if migration 0021 has not - /// been applied there is no trigger enforcing it — and an LSN probe + /// been applied there is no trigger enforcing it — and a heartbeat probe /// would open the fence over an unenforced floor. So the probe is gated /// on an unconditional two-part verification against the live schema: /// catalog shape ([`replica_fence::verify_floor_guard_catalog`]) and @@ -449,14 +552,13 @@ impl Db { /// stays closed: every cursor page routes to the writer. The relay keeps /// serving — degraded capacity, never holes. pub async fn spawn_fence_probe(&self) -> Result { - let Some(read_pool) = &self.read_pool else { + if self.read_pool.is_none() { return Ok(false); - }; + } replica_fence::verify_floor_guard_catalog(&self.pool).await?; replica_fence::verify_floor_guard_behavior(&self.pool).await?; tokio::spawn(replica_fence::run_probe( self.pool.clone(), - read_pool.clone(), std::sync::Arc::clone(&self.fence), )); Ok(true) @@ -478,6 +580,69 @@ impl Db { self.read_pool.is_some() } + /// Acquire a reader session and complete the connection-local half of + /// the fence proof: observe the heartbeat token/epoch **on that exact + /// session** and resolve it against the retained ring. Returns the + /// proved session together with the strongest [`replica_fence::TokenEntry`] + /// its observation supports, or the fail-closed reason for route + /// metrics. + /// + /// Everything but `Ok` fails closed — acquire failure, missing + /// heartbeat row (migration not yet replayed there), observation error, + /// epoch mismatch, or a token below every retained entry all route the + /// request to the writer. + async fn proved_reader( + &self, + read_pool: &PgPool, + ) -> std::result::Result< + ( + sqlx::pool::PoolConnection, + replica_fence::TokenEntry, + ), + &'static str, + > { + let mut conn = match read_pool.acquire().await { + Ok(conn) => conn, + Err(e) => { + tracing::warn!(error = %e, "reader acquire failed; routing to writer"); + return Err("reader_validation_error"); + } + }; + let obs = match replica_fence::observe_heartbeat(&mut conn).await { + Ok(Some(observation)) => observation, + Ok(None) => return Err("reader_validation_error"), + Err(e) => { + tracing::warn!(error = %e, "heartbeat observation failed; routing to writer"); + return Err("reader_validation_error"); + } + }; + match self.fence.resolve(obs.token, obs.epoch) { + replica_fence::ResolveOutcome::Proved(entry) => { + tracing::debug!( + token = obs.token, + proved_token = entry.token, + backend = %obs.backend, + "reader session proved fence coverage" + ); + Ok((conn, entry)) + } + replica_fence::ResolveOutcome::EpochMismatch => Err("reader_validation_error"), + replica_fence::ResolveOutcome::TokenBehind => Err("reader_token_behind"), + } + } + + /// Record one route decision (Rev 2 observability): which path, where it + /// went, and why. + fn record_route(path: &'static str, decision: &'static str, reason: &'static str) { + metrics::counter!( + "buzz_db_route_decision", + "path" => path, + "decision" => decision, + "reason" => reason, + ) + .increment(1); + } + /// Run pending database migrations. pub async fn migrate(&self) -> Result<()> { migration::run_migrations(&self.pool).await @@ -1990,19 +2155,25 @@ impl Db { /// Fetch replies under a root event. /// - /// Routing: the head fetch (`cursor: None`) always reads the writer. - /// Cursor-bearing pages may read the replica pool when one is configured - /// AND the freshness fence is open ([`replica_fence`]). Thread pagination - /// walks forward from oldest to newest, so a replica page is served only - /// when it is provably complete: + /// Routing mirrors [`Db::get_channel_window_with_session`]: a head + /// fetch (`cursor: None`) is Predicate A (bounded staleness, gated by + /// the default-off head budget); cursor pages are Predicate B + /// (completeness). Thread pagination walks **forward** from oldest to + /// newest, so a cursor carries no upper bound — instead the served page + /// is post-verified against the wall the serving session proved: /// /// - an under-`limit` page is a candidate terminal page — the client /// treats it as EOF, so it is re-run on the writer to keep the EOF /// decision authoritative (a lagged replica could truncate the tail); - /// - a full page whose newest row exceeds the fence could straddle a row - /// the replica has not replayed (commit order is not `created_at` - /// order), so it is also re-run on the writer. Only a full page that - /// sits entirely at or below the fence is served from the replica. + /// - a full page whose newest row exceeds the proved fence wall could + /// straddle a row the session has not replayed (commit order is not + /// `created_at` order), so it is also re-run on the writer. Only a + /// full page that sits entirely at or below the proved wall is served + /// from the replica. + /// + /// A head fetch routed under Predicate A skips the re-run: bounded + /// staleness (missing at most the freshest budget-window of replies) is + /// exactly the semantic the head gate accepts. pub async fn get_thread_replies( &self, community_id: CommunityId, @@ -2011,9 +2182,13 @@ impl Db { limit: u32, cursor: Option<&[u8]>, ) -> Result> { - if cursor.is_some() && self.has_read_pool() && self.fence.verified_through().is_some() { - let replies = thread::get_thread_replies( - self.read(), + let (path, predicate): (&'static str, RoutePredicate) = match cursor { + Some(_) => ("thread_cursor", RoutePredicate::Cursor(None)), + None => ("thread_head", RoutePredicate::Head), + }; + if let RouteDecision::Replica(mut conn, entry) = self.route_read(path, predicate).await { + let replies = thread::get_thread_replies_on( + &mut conn, community_id, root_event_id, depth_limit, @@ -2021,15 +2196,20 @@ impl Db { cursor, ) .await?; + if cursor.is_none() { + // Predicate A: bounded-stale head page, served as proved. + return Ok(replies); + } let full = replies.len() >= limit as usize; let below_fence = replies .last() - .is_some_and(|tail| self.fence.covers(tail.created_at)); + .is_some_and(|tail| tail.created_at <= entry.fence_wall); if full && below_fence { return Ok(replies); } - // Candidate terminal page, or page reaching above the fence — - // verify against the writer. + // Candidate terminal page, or page reaching above the proved + // wall — verify against the writer. + Self::record_route("thread_eof", "writer", "stale"); } thread::get_thread_replies( &self.pool, @@ -2053,15 +2233,8 @@ impl Db { /// One channel window: top-level rows + summaries + server `has_more`. /// - /// Routing: the head fetch (`cursor: None`) always reads the writer — it - /// must include just-committed events. A cursor-bearing page scrolls - /// *backward* into history bounded above by the cursor timestamp - /// (`created_at < ts`, or `= ts` with the id tiebreak), so it may read - /// the replica when one is configured AND the freshness fence covers the - /// cursor timestamp: every row the page could contain is then provably - /// replayed on the replica ([`replica_fence`]). Pages whose cursor - /// reaches above the fence — the freshest sliver of history — stay on - /// the writer. + /// Convenience wrapper over [`Db::get_channel_window_with_session`] for + /// callers with no follow-up queries; the serving session is released. pub async fn get_channel_window( &self, community_id: CommunityId, @@ -2070,11 +2243,153 @@ impl Db { cursor: Option<(DateTime, Vec)>, kind_filter: Option<&[u32]>, ) -> Result { - let pool = match &cursor { - Some((ts, _)) if self.has_read_pool() && self.fence.covers(*ts) => self.read(), - _ => &self.pool, + self.get_channel_window_with_session(community_id, channel_id, limit, cursor, kind_filter) + .await + .map(|(window, _session)| window) + } + + /// [`Db::get_channel_window`], additionally returning the session that + /// served the page so request-scoped follow-ups (the aux closure) run on + /// the same proved connection. + /// + /// Routing: + /// + /// - **Cursor page** (Predicate B — completeness): scrolls *backward* + /// into history bounded above by the cursor timestamp (`created_at < + /// ts`, or `= ts` with the id tiebreak), so it may be served by a + /// replica session when one is configured AND that session **proves** + /// coverage of the cursor timestamp: the heartbeat token/epoch is + /// observed on the exact connection that will serve the page and + /// resolved against the fence's retained ring ([`replica_fence`]). + /// - **Head fetch** (Predicate A — bounded staleness): served by a + /// proved replica session only when the head gate is configured + /// ([`DbConfig::replica_head_max_age_secs`], default off) and the + /// proved entry is within the budget. This trades a bounded staleness + /// window (budget plus probe cadence) on the GET leg for writer + /// offload; the WS `since`-overlap union covers fresh events. + /// + /// Every failure fails closed to the writer and is recorded in + /// `buzz_db_route_decision`. + pub async fn get_channel_window_with_session( + &self, + community_id: CommunityId, + channel_id: Uuid, + limit: u32, + cursor: Option<(DateTime, Vec)>, + kind_filter: Option<&[u32]>, + ) -> Result<(thread::ChannelWindow, ReadSession)> { + let path: &'static str = if cursor.is_some() { + "channel_cursor" + } else { + "channel_head" }; - thread::get_channel_window(pool, community_id, channel_id, limit, cursor, kind_filter).await + match self + .route_read(path, RoutePredicate::from_channel_cursor(&cursor)) + .await + { + RouteDecision::Replica(mut conn, _entry) => { + let window = thread::get_channel_window_on( + &mut conn, + community_id, + channel_id, + limit, + cursor, + kind_filter, + ) + .await?; + Ok(( + window, + ReadSession { + inner: ReadSessionInner::Replica(conn), + }, + )) + } + RouteDecision::Writer => { + let window = thread::get_channel_window( + &self.pool, + community_id, + channel_id, + limit, + cursor, + kind_filter, + ) + .await?; + Ok(( + window, + ReadSession { + inner: ReadSessionInner::Writer(self.pool.clone()), + }, + )) + } + } + } + + /// Shared route decision for one read: evaluate the predicate against a + /// proved reader session and record the decision. Fail closed to the + /// writer everywhere. + async fn route_read(&self, path: &'static str, predicate: RoutePredicate) -> RouteDecision { + let Some(read_pool) = &self.read_pool else { + Self::record_route(path, "writer", "disabled"); + return RouteDecision::Writer; + }; + // Cheap prechecks on the shared ring before spending a reader + // checkout; the connection-local observation still has to prove it. + let Some(newest) = self.fence.newest() else { + Self::record_route(path, "writer", "uninitialized"); + return RouteDecision::Writer; + }; + match &predicate { + // Predicate B precheck: the newest wall must cover the cursor + // (when the caller has an upper bound at all). + RoutePredicate::Cursor(Some(ts)) => { + if *ts > newest.fence_wall { + Self::record_route(path, "writer", "stale"); + return RouteDecision::Writer; + } + } + RoutePredicate::Cursor(None) => {} + // Predicate A precheck: head routing must be enabled, and the + // newest entry within budget (else no proved entry can be). + RoutePredicate::Head => match self.replica_head_max_age { + Some(budget) if newest.committed_at.elapsed() <= budget => {} + Some(_) => { + Self::record_route(path, "writer", "stale"); + return RouteDecision::Writer; + } + None => { + Self::record_route(path, "writer", "disabled"); + return RouteDecision::Writer; + } + }, + } + match self.proved_reader(read_pool).await { + Ok((conn, entry)) => { + let (holds, reason): (bool, &'static str) = match &predicate { + RoutePredicate::Cursor(Some(ts)) => (*ts <= entry.fence_wall, "covered"), + // No upper bound: the caller post-verifies the served + // rows against the proved wall. + RoutePredicate::Cursor(None) => (true, "covered"), + RoutePredicate::Head => ( + self.replica_head_max_age + .is_some_and(|budget| entry.committed_at.elapsed() <= budget), + "fresh", + ), + }; + if holds { + Self::record_route(path, "replica", reason); + RouteDecision::Replica(conn, entry) + } else { + // The session proves an older entry than the predicate + // needs (replication lag) — fail closed. + Self::record_route(path, "writer", "stale"); + RouteDecision::Writer + } + } + Err(reason) => { + Self::record_route(path, "writer", reason); + RouteDecision::Writer + } + } } /// Look up a single thread_metadata row by event_id. @@ -5436,6 +5751,20 @@ mod tests { assert!(db.read_pool_stats().is_none()); } + #[test] + fn head_budget_zero_disables_and_large_values_clamp_to_staleness() { + assert_eq!(head_budget_from_secs(0), None, "0 = head routing off"); + assert_eq!( + head_budget_from_secs(5), + Some(std::time::Duration::from_secs(5)) + ); + assert_eq!( + head_budget_from_secs(10_000), + Some(replica_fence::FENCE_STALENESS), + "budgets above the staleness gate clamp to it" + ); + } + /// Channel window: head fetch (no cursor) reads the WRITER; cursor pages /// read the REPLICA. Divergent fixtures prove which pool served each. #[tokio::test] @@ -5519,6 +5848,90 @@ mod tests { drop_scratch_db(&admin, writer, &wname).await; } + /// Head gate (Predicate A): with the budget unset, a head fetch reads + /// the writer even over an open fence; with a budget set and a fresh + /// proved entry, the head page is served by the replica session + /// (bounded staleness accepted); with a budget the fence entry exceeds, + /// the head page falls back to the writer. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn head_fetch_routes_by_configured_budget() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (writer, wname) = create_scratch_db(&admin, "head_w").await; + let (replica, rname) = create_scratch_db(&admin, "head_r").await; + + let author = nostr::Keys::generate(); + let community = Uuid::new_v4(); + let channel = Uuid::new_v4(); + seed_community_channel(&writer, community, channel, &author).await; + seed_community_channel(&replica, community, channel, &author).await; + + let base = 1_700_000_000u64; + let shared = signed_event_at(&author, "shared", base); + for pool in [&writer, &replica] { + insert_top_level(pool, community, channel, &shared).await; + } + // Divergent heads prove which pool served the fetch. + let fresh = signed_event_at(&author, "fresh-writer-only", base + 30); + insert_top_level(&writer, community, channel, &fresh).await; + let marker = signed_event_at(&author, "replica-only-marker", base + 20); + insert_top_level(&replica, community, channel, &marker).await; + + let mut db = Db::from_pools(writer.clone(), replica.clone()); + db.fence().force_open_for_tests(chrono::Utc::now()); + let cid = CommunityId::from_uuid(community); + let head_contents = |w: &thread::ChannelWindow| -> Vec { + w.rows + .iter() + .map(|r| r.stored_event.event.content.clone()) + .collect() + }; + + // Budget unset (rollout default): head → writer, fence open or not. + let head = db + .get_channel_window(cid, channel, 2, None, None) + .await + .expect("head, gate off"); + assert_eq!( + head_contents(&head), + vec!["fresh-writer-only".to_string(), "shared".to_string()], + "head routing must default off" + ); + + // Budget set, entry fresh (just recorded): head → replica. + db.set_replica_head_max_age_for_tests(Some(std::time::Duration::from_secs(5))); + let head = db + .get_channel_window(cid, channel, 2, None, None) + .await + .expect("head, gate on"); + assert_eq!( + head_contents(&head), + vec!["replica-only-marker".to_string(), "shared".to_string()], + "a fresh proved entry within budget must serve the head from the replica" + ); + + // Entry older than the budget: head falls back to the writer. + db.fence().close(); + db.fence().force_open_for_tests_at( + chrono::Utc::now(), + std::time::Instant::now() - std::time::Duration::from_secs(10), + ); + let head = db + .get_channel_window(cid, channel, 2, None, None) + .await + .expect("head, entry too old"); + assert_eq!( + head_contents(&head), + vec!["fresh-writer-only".to_string(), "shared".to_string()], + "an over-budget entry must fail the head gate closed" + ); + + drop_scratch_db(&admin, replica, &rname).await; + drop_scratch_db(&admin, writer, &wname).await; + } + /// Thread replies: head fetch reads the writer; a FULL cursor page is /// served by the replica; an UNDER-limit cursor page (candidate terminal /// page) is re-run on the writer so a lagged replica can never truncate @@ -6012,21 +6425,39 @@ mod tests { let base = admin_url().await; let idx = base.rfind('/').expect("db url has a path segment"); - let db = Db::new(&DbConfig { - database_url: format!("{}/{}", &base[..idx], wname), - read_database_url: Some(format!("{}/{}", &base[..idx], rname)), + let writer_url = format!("{}/{}", &base[..idx], wname); + let replica_url = format!("{}/{}", &base[..idx], rname); + + // Healthy schema: verification passes, probe starts. A SEPARATE Db + // instance, because its background probe legitimately opens its own + // fence (the heartbeat probe is writer-side only) — the refusal + // assertions below must run against a fence whose spawns were all + // refused. + let db_healthy = Db::new(&DbConfig { + database_url: writer_url.clone(), + read_database_url: Some(replica_url.clone()), max_connections: 2, ..DbConfig::default() }) .await .expect("connect armed Db with replica"); - - // Healthy schema: verification passes, probe starts. assert!( - db.spawn_fence_probe().await.expect("verification passes"), + db_healthy + .spawn_fence_probe() + .await + .expect("verification passes"), "probe must start on a verified schema" ); + let db = Db::new(&DbConfig { + database_url: writer_url, + read_database_url: Some(replica_url), + max_connections: 2, + ..DbConfig::default() + }) + .await + .expect("connect armed Db with replica"); + // Sabotage A: catalog-shaped no-op — same trigger, gutted function // body. Catalog check alone would pass; behavior check must refuse. sqlx::query( @@ -6066,6 +6497,10 @@ mod tests { "fence must remain closed when verification refuses the probe" ); + db_healthy.pool.close().await; + if let Some(rp) = &db_healthy.read_pool { + rp.close().await; + } db.pool.close().await; if let Some(rp) = &db.read_pool { rp.close().await; diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 0d84644b1d..6985916bba 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -1161,7 +1161,7 @@ mod tests { run_migrations(&pool) .await .expect("retry succeeds after operator repair"); - assert_eq!(applied_versions(&pool).await.last().copied(), Some(25)); + assert_eq!(applied_versions(&pool).await.last().copied(), Some(26)); } #[tokio::test] diff --git a/crates/buzz-db/src/replica_fence.rs b/crates/buzz-db/src/replica_fence.rs index 03bc1c77f0..b56c359115 100644 --- a/crates/buzz-db/src/replica_fence.rs +++ b/crates/buzz-db/src/replica_fence.rs @@ -9,38 +9,55 @@ //! (`clock_timestamp()`, evaluated inside commit processing). Enforcement //! is armed per session via the `buzz.created_at_floor` GUC, which the //! relay's writer pool sets on every connection. -//! 2. **Ordered LSN handshake** (this module): on one pinned writer -//! connection, three separately-awaited statements sample +//! 2. **Ordered heartbeat handshake** (this module): on one pinned writer +//! connection, separately-awaited statements sample //! `S = clock_timestamp()`, then scan `pg_stat_activity` for the oldest -//! open transaction, then capture `L = pg_current_wal_lsn()` **last**. -//! Once the replica reports `pg_last_wal_replay_lsn() >= L`, every -//! transaction partitions into exactly three buckets: -//! (a) finished before the activity scan — its commit WAL precedes `L`, -//! so the replica has replayed it; +//! open transaction, then — **last** — commit heartbeat token `M` via a +//! single-row `UPDATE replica_heartbeat ... RETURNING token, epoch` +//! (migration 0026). Because the single-row UPDATE serializes all pods' +//! probes, tokens are globally commit-ordered. A reader **session** that +//! observes `token >= M` on its own connection has, by WAL/storage replay +//! order, also replayed every commit that preceded M's commit; every +//! transaction then partitions into exactly three buckets: +//! (a) finished before the activity scan — its commit precedes `M`'s +//! commit, so the replica session has replayed it; //! (b) open at the activity scan — represented by `xact_start`, so it is //! bounded by the `oldest_xact_start` term; //! (c) started after the activity scan — its deferred floor guard runs //! after `S`, so it cannot commit a row with //! `created_at < S - floor`. -//! There is no fourth bucket. The fence therefore advances to -//! `min(oldest_xact_start, S) - floor - clock_margin`, and every -//! channel-window row with `created_at <= fence` is on the replica. +//! There is no fourth bucket. Each committed token `M` therefore proves a +//! **fence wall** of `min(oldest_xact_start, S) - floor - clock_margin`: +//! every channel-window row with `created_at <= fence_wall(M)` is present +//! on any reader session observing `token >= M`. +//! +//! Unlike the previous WAL-LSN observation (`pg_last_wal_replay_lsn()`, which +//! Aurora reader endpoints hide), the token observation is portable and — +//! critically — **connection-local**: the proof binds to the exact reader +//! session that will serve the page, never to a different pooled session +//! (readers behind one endpoint may sit at different replay positions). +//! An observed token lower than the newest retained `M` is ordinary +//! replication lag, not a fault; the resolver simply proves from an older +//! retained `M`. Regression detection is writer-side only: a non-monotonic +//! `RETURNING token` or an epoch change (restore/re-seed) clears the retained +//! ring, so no stale entry can masquerade as fresh coverage. //! //! Everything fails **closed**: probe errors, masked `pg_stat_activity` -//! visibility, NULL/absent replica LSN (Aurora observability differences), -//! non-advancing replay, or probe staleness all close the fence, which routes -//! all reads back to the writer — degraded capacity, never holes. +//! visibility, an unreadable heartbeat row on the reader session, an epoch +//! mismatch, or an observed token below every retained entry all route the +//! request back to the writer — degraded capacity, never holes. //! //! Operational bypasses (sessions without the GUC, `session_replication_role //! = replica` restores) are outside the proof by design and require holding //! the fence closed for their duration; see `migrations/0021`. -use std::sync::atomic::{AtomicI64, Ordering}; -use std::sync::Arc; -use std::time::Duration; +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use chrono::{DateTime, Utc}; -use sqlx::{PgPool, Row}; +use sqlx::{PgConnection, PgPool, Row}; +use uuid::Uuid; /// Seconds of `created_at` history the commit-time floor guard tolerates. /// @@ -58,79 +75,220 @@ pub const CREATED_AT_FLOOR_SECS: i64 = 960; /// between machines. pub const FENCE_CLOCK_MARGIN_SECS: i64 = 5; -/// How often the probe samples the writer and checks the replica. -pub const PROBE_INTERVAL: Duration = Duration::from_secs(5); +/// How often the probe samples the writer and commits a heartbeat token. +/// +/// Rev 2 sets ~1s: the head-routing predicate (`covered_age <= B`, B +/// defaulting to 5s) needs the cadence well under the budget, or eligibility +/// flaps between beats. Cost is one single-row UPDATE tuple of WAL per beat +/// per pod. +pub const PROBE_INTERVAL: Duration = Duration::from_secs(1); -/// A fence older than this is stale: the probe has stopped confirming -/// freshness and the fence closes until a new handshake completes. +/// A fence whose newest entry is older than this is stale: the probe has +/// stopped committing tokens and routing eligibility closes until a new +/// handshake completes. +/// +/// Note this is an availability hygiene gate, not a soundness requirement: +/// a retained entry's proof (`token >= M` on a session implies every row +/// `<= fence_wall(M)` is present there) never decays. Closing on staleness +/// just stops spending reader checkouts once the probe is evidently dead. pub const FENCE_STALENESS: Duration = Duration::from_secs(30); -/// Sentinel: fence closed (no verified replica coverage). -const CLOSED: i64 = i64::MIN; +/// How many `(token, fence_wall)` entries the fence retains. At one probe +/// per [`PROBE_INTERVAL`] this is ~2 minutes of history — a reader session +/// lagging further than that behind the newest token fails closed +/// (routes to the writer) rather than proving from thin air. Aurora reader +/// lag is typically tens of milliseconds; a reader minutes behind is a +/// fault, not a routing candidate. +const RING_CAPACITY: usize = 120; -/// Shared fence state. `Db` holds an `Arc` of this; the probe task advances -/// it and cursor routing consults it. -#[derive(Debug)] +/// One retained heartbeat observation: proof material for reader sessions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TokenEntry { + /// The committed heartbeat token `M`. + pub token: i64, + /// Monotonic instant captured just before `M` was committed. `elapsed()` + /// bounds (from above) how old a session observing `token >= M` can be — + /// the freshness term of the head-routing predicate. + pub committed_at: Instant, + /// `min(oldest_xact_start, S) - floor - clock_margin` for `M`'s + /// handshake: every channel-window row with `created_at <= fence_wall` + /// is present on any session observing `token >= M`. + pub fence_wall: DateTime, +} + +#[derive(Debug, Default)] +struct FenceInner { + /// Epoch the retained ring belongs to. `None` until the first probe — + /// or after the test hook, whose injected entry deliberately bypasses + /// the epoch comparison in [`ReplicaFence::resolve`]. + epoch: Option, + /// Retained entries in strictly increasing token order. + ring: VecDeque, +} + +/// Outcome of recording one probe sample. +#[derive(Debug, PartialEq, Eq)] +pub enum RecordOutcome { + /// Entry retained; proofs may cite it. + Recorded, + /// The token went backwards within the same epoch — a restore that kept + /// the old epoch. The ring was cleared and the entry discarded; the + /// probe must rotate the epoch before recording again (a reader still on + /// the pre-rewind timeline could otherwise observe a *higher* token that + /// proves nothing about the new timeline). + TokenRegression, +} + +/// Outcome of resolving one reader-session observation against the ring. +/// Everything but [`ResolveOutcome::Proved`] fails closed (routes to the +/// writer); the variants exist so route metrics can name the reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResolveOutcome { + /// The observation proves this retained entry. + Proved(TokenEntry), + /// The observed epoch is not the ring's epoch — the session is on a + /// different timeline (restore) or the ring was rotated under it. + EpochMismatch, + /// The observed token is below every retained entry: the reader lags + /// further than the ring's history (or the ring is empty). + TokenBehind, +} + +impl ResolveOutcome { + /// The proved entry, if any. + pub fn proved(self) -> Option { + match self { + ResolveOutcome::Proved(entry) => Some(entry), + _ => None, + } + } +} + +/// Shared fence state. `Db` holds an `Arc` of this; the probe task records +/// entries and per-request routing resolves proofs against it. +#[derive(Debug, Default)] pub struct ReplicaFence { - /// Unix micros of the newest verified-complete timestamp, or `CLOSED`. - fence_micros: AtomicI64, - /// Unix micros when the fence was last advanced (staleness check). - updated_micros: AtomicI64, + inner: Mutex, } impl ReplicaFence { - /// A new fence, initially closed. + /// A new fence, initially closed (empty ring). pub fn new() -> Self { - Self { - fence_micros: AtomicI64::new(CLOSED), - updated_micros: AtomicI64::new(CLOSED), - } + Self::default() } - /// Close the fence: all cursor reads route to the writer. + /// Close the fence: drop all retained proofs; reads route to the writer. pub fn close(&self) { - self.fence_micros.store(CLOSED, Ordering::Relaxed); + let mut inner = self.inner.lock().expect("fence lock poisoned"); + inner.ring.clear(); + } + + /// Record one probe sample. Epoch changes (re-seed) clear the ring and + /// start a new one under the observed epoch — sound, because an entry + /// only proves commits on its own timeline and readers must match the + /// epoch to cite it. A same-epoch token regression is the unsafe case: + /// see [`RecordOutcome::TokenRegression`]. + pub fn record( + &self, + token: i64, + epoch: Uuid, + committed_at: Instant, + fence_wall: DateTime, + ) -> RecordOutcome { + let mut inner = self.inner.lock().expect("fence lock poisoned"); + if inner.epoch != Some(epoch) { + inner.ring.clear(); + inner.epoch = Some(epoch); + } else if inner.ring.back().is_some_and(|last| token <= last.token) { + inner.ring.clear(); + return RecordOutcome::TokenRegression; + } + if inner.ring.len() == RING_CAPACITY { + inner.ring.pop_front(); + } + inner.ring.push_back(TokenEntry { + token, + committed_at, + fence_wall, + }); + RecordOutcome::Recorded } - fn advance(&self, fence: DateTime) { - self.fence_micros - .store(fence.timestamp_micros(), Ordering::Relaxed); - self.updated_micros - .store(Utc::now().timestamp_micros(), Ordering::Relaxed); + /// Resolve the strongest proof a reader session's observation supports: + /// the greatest retained entry with `entry.token <= observed_token`, + /// provided the observed epoch matches the ring's. Non-`Proved` outcomes + /// fail closed; they are distinguished only for route metrics. + pub fn resolve(&self, observed_token: i64, observed_epoch: Uuid) -> ResolveOutcome { + let inner = self.inner.lock().expect("fence lock poisoned"); + match inner.epoch { + Some(e) if e != observed_epoch => return ResolveOutcome::EpochMismatch, + // `None` with a non-empty ring only happens via the test hook; + // the epoch comparison is deliberately skipped there. + _ => {} + } + inner + .ring + .iter() + .rev() + .find(|entry| entry.token <= observed_token) + .copied() + .map_or(ResolveOutcome::TokenBehind, ResolveOutcome::Proved) } - /// The current fence, or `None` when closed or stale. + /// The newest retained entry, staleness-gated. Used as the cheap + /// pre-check before spending a reader checkout, and for observability. + pub fn newest(&self) -> Option { + let inner = self.inner.lock().expect("fence lock poisoned"); + inner + .ring + .back() + .filter(|entry| entry.committed_at.elapsed() <= FENCE_STALENESS) + .copied() + } + + /// Age of the newest retained entry, ungated (observability: how long + /// since the probe last committed a token). + pub fn heartbeat_age(&self) -> Option { + let inner = self.inner.lock().expect("fence lock poisoned"); + inner.ring.back().map(|entry| entry.committed_at.elapsed()) + } + + /// The newest fence wall, or `None` when closed or stale. /// - /// Rows with `created_at <= fence` are verified present on the replica. + /// Rows with `created_at <= fence` are verified present on a reader + /// session that proves the newest entry; whether a *given* session does + /// is decided per request via [`ReplicaFence::resolve`]. pub fn verified_through(&self) -> Option> { - let raw = self.fence_micros.load(Ordering::Relaxed); - if raw == CLOSED { - return None; - } - let updated = self.updated_micros.load(Ordering::Relaxed); - let age_micros = Utc::now().timestamp_micros().saturating_sub(updated); - if age_micros > FENCE_STALENESS.as_micros() as i64 { - return None; - } - DateTime::from_timestamp_micros(raw) + self.newest().map(|entry| entry.fence_wall) } - /// Whether the replica verifiably holds every channel-window row at or - /// before `ts`. + /// Whether some retained entry's wall covers `ts` — the cheap routing + /// pre-check (the connection-local observation still has to prove it). pub fn covers(&self, ts: DateTime) -> bool { self.verified_through().is_some_and(|fence| ts <= fence) } /// Test hook: force the fence open through `ts` without a probe. - /// Used by routing tests that stand up a divergent fake replica. + /// Injects an entry any observed token satisfies (`i64::MIN`) with no + /// epoch recorded, so the epoch comparison is bypassed — routing tests + /// stand up a divergent fake replica whose heartbeat epoch differs from + /// the writer's. pub fn force_open_for_tests(&self, ts: DateTime) { - self.advance(ts); + self.force_open_for_tests_at(ts, Instant::now()); } -} -impl Default for ReplicaFence { - fn default() -> Self { - Self::new() + /// [`ReplicaFence::force_open_for_tests`] with an explicit commit + /// instant, for pinning age-gated behavior (head-budget and staleness + /// tests inject entries "committed" in the past). + pub fn force_open_for_tests_at(&self, ts: DateTime, committed_at: Instant) { + let mut inner = self.inner.lock().expect("fence lock poisoned"); + inner.epoch = None; + inner.ring.clear(); + inner.ring.push_back(TokenEntry { + token: i64::MIN, + committed_at, + fence_wall: ts, + }); } } @@ -332,15 +490,20 @@ struct WriterSample { /// Oldest open transaction among other client backends at scan time, /// or `None` when no transaction was open. oldest_xact_start: Option>, - /// `L`: writer `pg_current_wal_lsn()` captured last, as text. - wal_lsn: String, + /// `M`: the heartbeat token committed **last**, after the scan. + token: i64, + /// Heartbeat epoch returned with `M`. + epoch: Uuid, + /// Monotonic instant captured immediately before committing `M` — an + /// upper bound on how old a session observing `token >= M` can be. + committed_at: Instant, } /// Errors that close the fence. All variants are logged and treated /// identically: fail closed. #[derive(Debug, thiserror::Error)] pub enum ProbeError { - /// A probe query against writer or replica failed. + /// A probe query against the writer failed. #[error("writer probe query failed: {0}")] Writer(#[from] sqlx::Error), /// `pg_stat_activity` hid state for another backend that could hold an @@ -353,14 +516,15 @@ pub enum ProbeError { /// Number of other client backends with masked/unknown state. masked: i64, }, - /// The replica returned NULL for the replay-LSN comparison. - #[error("replica did not report a comparable replay LSN")] - ReplicaLsnUnavailable, + /// The single heartbeat row (migration 0026) is missing on the writer. + #[error("replica_heartbeat row missing on the writer — migration 0026 not applied?")] + HeartbeatRowMissing, } -/// Take one ordered writer sample: S, then activity scan, then L **last**. +/// Take one ordered writer sample: S, then activity scan, then commit the +/// heartbeat token **last**. /// -/// The three statements are separately awaited on a single pinned connection; +/// The statements are separately awaited on a single pinned connection; /// a single SELECT would not guarantee evaluation order across the /// subexpressions, reopening the race this ordering exists to close. async fn sample_writer(writer: &PgPool) -> Result { @@ -393,8 +557,8 @@ async fn sample_writer(writer: &PgPool) -> Result { // // Prepared transactions (2PC) are a bucket of their own: while // prepared they have left `pg_stat_activity` but can still commit - // after `L`. Their deferred floor guard already ran at PREPARE, so - // `pg_prepared_xacts.prepared` bounds their rows exactly like + // after the token. Their deferred floor guard already ran at PREPARE, + // so `pg_prepared_xacts.prepared` bounds their rows exactly like // `xact_start`; fold it into the same minimum. let row = sqlx::query( r#" @@ -423,80 +587,129 @@ async fn sample_writer(writer: &PgPool) -> Result { } let oldest_xact_start: Option> = row.get("oldest_xact_start"); - // 3. L last. - let wal_lsn: String = sqlx::query_scalar("SELECT pg_current_wal_lsn()::text") - .fetch_one(&mut *conn) - .await?; + // 3. Token commit LAST, on the same pinned connection. The single-row + // UPDATE serializes concurrent pods' probes, so RETURNING token is + // globally commit-ordered. `committed_at` is captured before the + // round trip so `elapsed()` over-estimates the observation's age — + // the conservative direction for the head-freshness bound. + let committed_at = Instant::now(); + let row = sqlx::query( + "UPDATE replica_heartbeat SET token = token + 1 WHERE id = 1 RETURNING token, epoch", + ) + .fetch_optional(&mut *conn) + .await? + .ok_or(ProbeError::HeartbeatRowMissing)?; Ok(WriterSample { sampled_at, oldest_xact_start, - wal_lsn, + token: row.get("token"), + epoch: row.get("epoch"), + committed_at, }) } -/// Whether the replica has replayed at least through `wal_lsn`. -/// -/// The comparison happens on the replica in pg_lsn domain. The -/// `pg_is_in_recovery()` gate is load-bearing: after crash recovery or -/// promotion a *primary* returns a static non-NULL `pg_last_wal_replay_lsn()` -/// rather than NULL, so NULL-checking alone would not reliably detect a -/// misrouted "replica" URL. Not-in-recovery, NULL replay LSN, or Aurora -/// hiding either is an error → fence closes. -async fn replica_covers(replica: &PgPool, wal_lsn: &str) -> Result { - let covered: Option = sqlx::query_scalar( - r#" - SELECT CASE - WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() >= $1::pg_lsn - ELSE NULL - END - "#, - ) - .bind(wal_lsn) - .fetch_one(replica) - .await?; - covered.ok_or(ProbeError::ReplicaLsnUnavailable) +/// The fence wall proved by one handshake: +/// `min(oldest_xact_start, S) - floor - clock_margin`. +fn fence_wall(sample_s: DateTime, oldest_xact_start: Option>) -> DateTime { + let lower = match oldest_xact_start { + Some(oldest) => oldest.min(sample_s), + None => sample_s, + }; + lower + - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS) + - chrono::Duration::seconds(FENCE_CLOCK_MARGIN_SECS) } -/// Run one full handshake and, on success, advance the fence. +/// Run one full handshake and record the resulting `(token, fence_wall)`. /// -/// Returns the new fence value for observability. `Ok(None)` means the -/// replica has not yet replayed past the sample; the fence is left as-is -/// (staleness will close it if this persists). -pub async fn probe_once( - writer: &PgPool, - replica: &PgPool, - fence: &ReplicaFence, -) -> Result>, ProbeError> { +/// On a same-epoch token regression (the writer was restored from a backup +/// that kept its epoch), the retained ring has already been cleared by +/// [`ReplicaFence::record`]; this additionally **rotates the epoch** on the +/// writer and records the rotated token, so a reader still serving the +/// pre-rewind timeline (whose old, higher token would otherwise satisfy +/// `token >= M`) fails the epoch check instead of proving stale coverage. +pub async fn probe_once(writer: &PgPool, fence: &ReplicaFence) -> Result { let sample = sample_writer(writer).await?; - if !replica_covers(replica, &sample.wal_lsn).await? { - return Ok(None); + let wall = fence_wall(sample.sampled_at, sample.oldest_xact_start); + match fence.record(sample.token, sample.epoch, sample.committed_at, wall) { + RecordOutcome::Recorded => Ok(TokenEntry { + token: sample.token, + committed_at: sample.committed_at, + fence_wall: wall, + }), + RecordOutcome::TokenRegression => { + tracing::warn!( + token = sample.token, + "replica heartbeat token regressed within its epoch (restore?); rotating epoch" + ); + // The rotation commit happens after this sample's activity scan, + // so the same three-bucket argument (and the same wall) holds + // for the rotated token. + let committed_at = Instant::now(); + let row = sqlx::query( + "UPDATE replica_heartbeat SET epoch = gen_random_uuid(), token = token + 1 \ + WHERE id = 1 RETURNING token, epoch", + ) + .fetch_optional(writer) + .await? + .ok_or(ProbeError::HeartbeatRowMissing)?; + let token: i64 = row.get("token"); + let epoch: Uuid = row.get("epoch"); + // A fresh epoch always clears and records; regression is + // impossible against an empty ring. + fence.record(token, epoch, committed_at, wall); + Ok(TokenEntry { + token, + committed_at, + fence_wall: wall, + }) + } } - let lower = match sample.oldest_xact_start { - Some(oldest) => oldest.min(sample.sampled_at), - None => sample.sampled_at, - }; - let new_fence = lower - - chrono::Duration::seconds(CREATED_AT_FLOOR_SECS) - - chrono::Duration::seconds(FENCE_CLOCK_MARGIN_SECS); - fence.advance(new_fence); - Ok(Some(new_fence)) } -/// Background probe loop: sample every `PROBE_INTERVAL`, close the fence on -/// any error. Runs for the life of the process. -pub async fn run_probe(writer: PgPool, replica: PgPool, fence: Arc) { +/// Observe the heartbeat on a specific reader session — the +/// connection-local half of the proof. Returns the observed token/epoch +/// plus the backend identity of the session (`inet_server_addr()`; +/// `"local"` on unix sockets) for route-decision evidence. `None` when the +/// row is missing there (migration not yet replayed): fail closed. +pub async fn observe_heartbeat( + conn: &mut PgConnection, +) -> Result, sqlx::Error> { + let row = sqlx::query( + "SELECT token, epoch, COALESCE(host(inet_server_addr()), 'local') AS backend \ + FROM replica_heartbeat WHERE id = 1", + ) + .fetch_optional(&mut *conn) + .await?; + Ok(row.map(|r| HeartbeatObservation { + token: r.get("token"), + epoch: r.get("epoch"), + backend: r.get("backend"), + })) +} + +/// One reader-session heartbeat observation (see [`observe_heartbeat`]). +#[derive(Debug, Clone)] +pub struct HeartbeatObservation { + /// The token the session has replayed through. + pub token: i64, + /// The epoch the session observes — must match the ring's. + pub epoch: Uuid, + /// Backend identity of the observed session, so live evidence records + /// which reader served both proof and page. + pub backend: String, +} + +/// Background probe loop: commit a heartbeat token every `PROBE_INTERVAL`; +/// close the fence on any error. Runs for the life of the process. +pub async fn run_probe(writer: PgPool, fence: Arc) { let mut interval = tokio::time::interval(PROBE_INTERVAL); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { interval.tick().await; - match probe_once(&writer, &replica, &fence).await { - Ok(Some(_)) => {} - Ok(None) => { - // Replica behind the sample: leave the fence; staleness - // closes it if the replica stays behind. - tracing::debug!("replica fence: replay behind writer sample"); - } + match probe_once(&writer, &fence).await { + Ok(_) => {} Err(e) => { fence.close(); tracing::warn!(error = %e, "replica fence probe failed; fence closed"); @@ -515,14 +728,50 @@ mod tests { std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| TEST_DB_URL.into()) } + /// A private scratch database with migrations applied: the probe tests + /// mutate the singleton heartbeat row (rewind/rotate), which must never + /// race the shared dev database or each other. + async fn scratch_db() -> (PgPool, PgPool, String) { + let admin = PgPool::connect(&test_db_url()) + .await + .expect("connect admin"); + let name = format!("fence_probe_{}", uuid::Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch db"); + let base = test_db_url(); + let idx = base.rfind('/').expect("db url has a path segment"); + let pool = PgPool::connect(&format!("{}/{}", &base[..idx], name)) + .await + .expect("connect scratch db"); + crate::migration::run_migrations(&pool) + .await + .expect("migrate scratch db"); + (admin, pool, name) + } + + async fn drop_scratch_db(admin: &PgPool, pool: PgPool, name: &str) { + pool.close().await; + let _ = sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {name} WITH (FORCE)" + ))) + .execute(admin) + .await; + } + #[test] - fn fence_starts_closed_and_opens_on_advance() { + fn fence_starts_closed_and_opens_on_record() { let fence = ReplicaFence::new(); assert!(fence.verified_through().is_none(), "must start closed"); assert!(!fence.covers(Utc::now() - chrono::Duration::days(365))); let ts = Utc::now(); - fence.advance(ts); + let epoch = Uuid::new_v4(); + assert_eq!( + fence.record(1, epoch, Instant::now(), ts), + RecordOutcome::Recorded + ); assert_eq!(fence.verified_through(), Some(ts)); assert!(fence.covers(ts - chrono::Duration::seconds(1))); assert!(fence.covers(ts), "boundary is inclusive"); @@ -537,19 +786,116 @@ mod tests { fn stale_fence_reads_as_closed() { let fence = ReplicaFence::new(); let ts = Utc::now(); - fence - .fence_micros - .store(ts.timestamp_micros(), Ordering::Relaxed); - // Last update older than the staleness budget. - let stale = (Utc::now() - - chrono::Duration::from_std(FENCE_STALENESS).expect("duration") - - chrono::Duration::seconds(1)) - .timestamp_micros(); - fence.updated_micros.store(stale, Ordering::Relaxed); + // Newest entry committed longer ago than the staleness budget. + let stale_instant = Instant::now() - (FENCE_STALENESS + Duration::from_secs(1)); + fence.record(1, Uuid::new_v4(), stale_instant, ts); assert!( fence.verified_through().is_none(), "a fence the probe stopped confirming must read as closed" ); + // heartbeat_age is deliberately ungated (observability). + assert!(fence.heartbeat_age().expect("entry retained") > FENCE_STALENESS); + } + + /// Resolve picks the greatest retained entry <= the observed token — + /// a lagged reader proves from an older wall, never from thin air. + #[test] + fn resolve_picks_greatest_retained_token_at_or_below_observation() { + let fence = ReplicaFence::new(); + let epoch = Uuid::new_v4(); + let base = Utc::now(); + for (token, secs) in [(10i64, 0i64), (20, 10), (30, 20)] { + fence.record( + token, + epoch, + Instant::now(), + base + chrono::Duration::seconds(secs), + ); + } + + // Exact hit. + assert_eq!(fence.resolve(20, epoch).proved().expect("proof").token, 20); + // Between entries: prove from the older one. + assert_eq!(fence.resolve(25, epoch).proved().expect("proof").token, 20); + // Ahead of everything retained: newest. + assert_eq!( + fence.resolve(1000, epoch).proved().expect("proof").token, + 30 + ); + // Behind everything retained: no proof. + assert_eq!( + fence.resolve(9, epoch), + ResolveOutcome::TokenBehind, + "token below ring fails closed" + ); + // Wrong epoch: no proof, regardless of token. + assert_eq!( + fence.resolve(1000, Uuid::new_v4()), + ResolveOutcome::EpochMismatch, + "epoch mismatch fails closed" + ); + } + + /// An epoch change clears the ring and starts a new one; a same-epoch + /// token regression clears the ring and reports the fault. + #[test] + fn record_epoch_change_resets_and_same_epoch_regression_fails() { + let fence = ReplicaFence::new(); + let epoch_a = Uuid::new_v4(); + let ts = Utc::now(); + fence.record(10, epoch_a, Instant::now(), ts); + fence.record(11, epoch_a, Instant::now(), ts); + + // New epoch, lower token: fine — new timeline, old proofs dropped. + let epoch_b = Uuid::new_v4(); + assert_eq!( + fence.record(3, epoch_b, Instant::now(), ts), + RecordOutcome::Recorded + ); + assert_eq!( + fence.resolve(11, epoch_a), + ResolveOutcome::EpochMismatch, + "entries from the old epoch must be gone" + ); + assert_eq!(fence.resolve(3, epoch_b).proved().expect("proof").token, 3); + + // Same epoch, non-increasing token: regression → cleared + reported. + assert_eq!( + fence.record(3, epoch_b, Instant::now(), ts), + RecordOutcome::TokenRegression + ); + assert!( + fence.verified_through().is_none(), + "ring cleared on regression" + ); + assert_eq!( + fence.resolve(i64::MAX, epoch_b), + ResolveOutcome::TokenBehind + ); + } + + /// The ring is bounded: old entries fall off and stop proving coverage. + #[test] + fn ring_capacity_evicts_oldest_entries() { + let fence = ReplicaFence::new(); + let epoch = Uuid::new_v4(); + let ts = Utc::now(); + for token in 0..(RING_CAPACITY as i64 + 10) { + fence.record(token, epoch, Instant::now(), ts); + } + assert_eq!( + fence.resolve(5, epoch), + ResolveOutcome::TokenBehind, + "evicted tokens must no longer prove coverage" + ); + assert_eq!( + fence + .resolve(i64::MAX, epoch) + .proved() + .expect("proof") + .token, + RING_CAPACITY as i64 + 9 + ); } /// The activity scan must (a) represent another session's open @@ -582,9 +928,14 @@ mod tests { oldest <= during.sampled_at, "xact_start precedes the sample that observed it" ); - // S is captured before the activity scan, L after: the sample's - // ordering invariant. + // S is captured before the activity scan, the token commit after: + // the sample's ordering invariant. assert!(during.sampled_at >= before.sampled_at); + assert!( + during.token > before.token, + "each sample must commit a strictly newer token" + ); + assert_eq!(during.epoch, before.epoch, "epoch is stable across samples"); tx.rollback().await.expect("rollback"); } @@ -641,21 +992,97 @@ mod tests { .expect("drop role"); } - /// A primary (non-replica) database returns NULL from - /// `pg_last_wal_replay_lsn()`; the probe must fail closed, never - /// synthesize freshness. This is also the Aurora-observability guard: - /// if the reader endpoint hides replay LSNs, routing stays writer-only. + /// End-to-end probe against a real database: each probe commits a + /// strictly newer token, records a retained entry, and a session on the + /// same database observes a token/epoch that resolves that entry. #[tokio::test] #[ignore = "requires Postgres"] - async fn probe_fails_closed_when_replica_lsn_unavailable() { - let pool = PgPool::connect(&test_db_url()).await.expect("connect"); + async fn probe_commits_tokens_and_sessions_prove_coverage() { + let (admin, pool, name) = scratch_db().await; let fence = ReplicaFence::new(); - fence.advance(Utc::now()); // pretend a previous handshake succeeded - // Using the primary as its own "replica": replay LSN is NULL. - let err = probe_once(&pool, &pool, &fence) + let first = probe_once(&pool, &fence).await.expect("first probe"); + let second = probe_once(&pool, &fence).await.expect("second probe"); + assert!(second.token > first.token, "tokens strictly increase"); + assert!( + second.fence_wall >= first.fence_wall + || second.fence_wall + > first.fence_wall - chrono::Duration::seconds(FENCE_CLOCK_MARGIN_SECS), + "walls advance with the clock (modulo an open transaction)" + ); + + // A "reader" session on the same database observes at least the + // second token and proves the newest retained entry. + let mut conn = pool.acquire().await.expect("reader conn"); + let obs = observe_heartbeat(&mut conn) .await - .expect_err("NULL replay LSN must be an error"); - assert!(matches!(err, ProbeError::ReplicaLsnUnavailable)); + .expect("observe") + .expect("heartbeat row present"); + assert!(obs.token >= second.token); + assert!(!obs.backend.is_empty(), "backend identity recorded"); + let proof = fence + .resolve(obs.token, obs.epoch) + .proved() + .expect("proof resolves"); + assert_eq!(proof.token, second.token, "newest retained entry cited"); + + // An epoch nobody committed proves nothing. + assert_eq!( + fence.resolve(obs.token, Uuid::new_v4()), + ResolveOutcome::EpochMismatch + ); + + drop(conn); + drop_scratch_db(&admin, pool, &name).await; + } + + /// A same-epoch token rewind on the writer (restore adversary) must not + /// leave proofs standing: the probe rotates the epoch, so a reader still + /// on the pre-rewind timeline — observing a *higher* token under the old + /// epoch — fails the epoch check instead of proving stale coverage. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn probe_rotates_epoch_on_same_epoch_token_regression() { + let (admin, pool, name) = scratch_db().await; + let fence = ReplicaFence::new(); + + let before = probe_once(&pool, &fence).await.expect("probe"); + let mut conn = pool.acquire().await.expect("conn"); + let old_epoch = observe_heartbeat(&mut conn) + .await + .expect("observe") + .expect("row") + .epoch; + + // Rewind the token in place, keeping the epoch: the restore shape. + sqlx::query("UPDATE replica_heartbeat SET token = 0 WHERE id = 1") + .execute(&pool) + .await + .expect("rewind token"); + + let after = probe_once(&pool, &fence).await.expect("recovery probe"); + // The pre-rewind observation must no longer prove anything. + assert_eq!( + fence.resolve(before.token, old_epoch), + ResolveOutcome::EpochMismatch, + "old-epoch observations must fail closed after rotation" + ); + // A fresh observation on the new timeline proves the rotated entry. + let obs = observe_heartbeat(&mut conn) + .await + .expect("observe") + .expect("row"); + assert_ne!(obs.epoch, old_epoch, "epoch rotated"); + assert_eq!( + fence + .resolve(obs.token, obs.epoch) + .proved() + .expect("proof") + .token, + after.token + ); + + drop(conn); + drop_scratch_db(&admin, pool, &name).await; } } diff --git a/crates/buzz-db/src/thread.rs b/crates/buzz-db/src/thread.rs index 3f92212dd5..007677e258 100644 --- a/crates/buzz-db/src/thread.rs +++ b/crates/buzz-db/src/thread.rs @@ -349,6 +349,30 @@ pub async fn get_thread_replies( depth_limit: Option, limit: u32, cursor: Option<&[u8]>, +) -> Result> { + let mut conn = pool.acquire().await?; + get_thread_replies_on( + &mut conn, + community_id, + root_event_id, + depth_limit, + limit, + cursor, + ) + .await +} + +/// [`get_thread_replies`] on a specific session — the replica-routing path +/// runs the page on the exact reader connection whose heartbeat observation +/// proved coverage (the proof is connection-local; a different pooled +/// session may sit at a different replay position). +pub(crate) async fn get_thread_replies_on( + conn: &mut sqlx::PgConnection, + community_id: CommunityId, + root_event_id: &[u8], + depth_limit: Option, + limit: u32, + cursor: Option<&[u8]>, ) -> Result> { // Decode cursor bytes -> keyset (timestamp, optional event_id) for the // WHERE condition. Layout: 8-byte BE i64 seconds, then the raw event_id. @@ -445,7 +469,7 @@ pub async fn get_thread_replies( } q = q.bind(limit as i32); - let rows = q.fetch_all(pool).await?; + let rows = q.fetch_all(&mut *conn).await?; let mut replies = Vec::with_capacity(rows.len()); for row in rows { @@ -569,6 +593,31 @@ pub async fn get_channel_window( limit: u32, cursor: Option<(DateTime, Vec)>, kind_filter: Option<&[u32]>, +) -> Result { + let mut conn = pool.acquire().await?; + get_channel_window_on( + &mut conn, + community_id, + channel_id, + limit, + cursor, + kind_filter, + ) + .await +} + +/// [`get_channel_window`] on a specific session — the replica-routing path +/// runs the page (and its participants batch) on the exact reader connection +/// whose heartbeat observation proved coverage (the proof is +/// connection-local; a different pooled session may sit at a different +/// replay position). +pub(crate) async fn get_channel_window_on( + conn: &mut sqlx::PgConnection, + community_id: CommunityId, + channel_id: Uuid, + limit: u32, + cursor: Option<(DateTime, Vec)>, + kind_filter: Option<&[u32]>, ) -> Result { let mut param_idx = 3u32; // $1 is community_id, $2 is channel_id let mut sql = String::from( @@ -637,7 +686,7 @@ pub async fn get_channel_window( // The +1 probe row is the server-internal has_more evidence. q = q.bind(limit as i64 + 1); - let mut db_rows = q.fetch_all(pool).await?; + let mut db_rows = q.fetch_all(&mut *conn).await?; let has_more = db_rows.len() > limit as usize; db_rows.truncate(limit as usize); @@ -722,7 +771,7 @@ pub async fn get_channel_window( ) .bind(community_id.as_uuid()) .bind(&roots) - .fetch_all(pool) + .fetch_all(&mut *conn) .await?; let mut by_root: std::collections::HashMap, Vec>> = diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 2e00d6bd2f..6cbd31202f 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -462,9 +462,9 @@ async fn handle_channel_window_filter( .as_ref() .map(|ks| ks.iter().map(|k| k.as_u16() as u32).collect()); - let window = state + let (window, mut session) = state .db - .get_channel_window( + .get_channel_window_with_session( tenant.community(), ch_id, limit, @@ -485,7 +485,11 @@ async fn handle_channel_window_filter( // 2. Aux closure: reactions/deletions/edits targeting retained rows, plus // deletions targeting those aux events (the transitive second hop). - // One round trip for the client instead of an #e fan-out. + // One round trip for the client instead of an #e fan-out. Runs on the + // SAME session that served the window: when the page came from a + // proved replica connection, hopping to another pooled session (which + // may sit at an older replay position) could drop aux rows the served + // page's proof already covers. if extension_flag(raw, "include_aux") && !row_ids_hex.is_empty() { let mut seen_aux: std::collections::HashSet = std::collections::HashSet::new(); @@ -495,8 +499,7 @@ async fn handle_channel_window_filter( aux_query.kinds = Some(hop_kinds.iter().map(|k| *k as i32).collect()); aux_query.e_tags = Some(std::mem::take(&mut hop_ids)); aux_query.limit = Some(1000); - let aux_events = state - .db + let aux_events = session .query_events(&aux_query) .await .map_err(|e| internal_error(&format!("window aux error: {e}")))?; diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 47030dcf3f..9323875b4a 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -56,6 +56,10 @@ pub struct Config { /// Optional read-replica connection URL (e.g. an Aurora `cluster-ro-` /// endpoint). Unset means all reads stay on the writer. pub read_database_url: Option, + /// Head-fetch replica budget `B` in seconds (`BUZZ_REPLICA_HEAD_MAX_AGE_SECS`). + /// `0` (the default) disables replica routing for head fetches; see + /// [`buzz_db::DbConfig::replica_head_max_age_secs`]. + pub replica_head_max_age_secs: u64, /// Redis connection URL used by the pub/sub manager. pub redis_url: String, /// Maximum connections in the shared Redis pool. Defaults to 16. @@ -415,6 +419,17 @@ impl Config { .map(|v| v.trim().to_string()) .filter(|v| !v.is_empty()); + // Head-fetch replica budget: 0 = off (the rollout default), so this + // is a non-negative parse, unlike `positive_u64_from_env`. + let replica_head_max_age_secs = match std::env::var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS") { + Ok(raw) => raw.trim().parse::().map_err(|_| { + ConfigError::InvalidValue( + "BUZZ_REPLICA_HEAD_MAX_AGE_SECS must be a non-negative integer".to_string(), + ) + })?, + Err(_) => 0, + }; + let redis_url = std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost:6379".to_string()); @@ -873,6 +888,7 @@ impl Config { bind_addr, database_url, read_database_url, + replica_head_max_age_secs, redis_url, redis_pool_size, relay_url, @@ -1037,6 +1053,44 @@ mod tests { ); } + #[test] + fn replica_head_max_age_defaults_off_and_rejects_junk() { + let _guard = ENV_MUTEX.lock().unwrap(); + let previous = std::env::var_os("BUZZ_REPLICA_HEAD_MAX_AGE_SECS"); + + std::env::remove_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS"); + let unset = Config::from_env() + .expect("config") + .replica_head_max_age_secs; + + std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", "5"); + let set = Config::from_env() + .expect("config") + .replica_head_max_age_secs; + + std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", "0"); + let zero = Config::from_env() + .expect("config") + .replica_head_max_age_secs; + + std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", "soon"); + let junk = Config::from_env(); + + if let Some(value) = previous { + std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", value); + } else { + std::env::remove_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS"); + } + + assert_eq!(unset, 0, "head routing must default off"); + assert_eq!(set, 5); + assert_eq!(zero, 0, "explicit 0 is off"); + assert!( + junk.is_err(), + "an unparsable budget must fail loudly, not silently disable" + ); + } + #[test] fn audit_logging_defaults_on_and_accepts_explicit_off() { let _guard = ENV_MUTEX.lock().unwrap(); diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 533428a4e8..6be21559a2 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -146,6 +146,7 @@ async fn main() -> anyhow::Result<()> { let db_config = DbConfig { database_url: config.database_url.clone(), read_database_url: config.read_database_url.clone(), + replica_head_max_age_secs: config.replica_head_max_age_secs, ..DbConfig::default() }; let db = Db::new(&db_config).await.map_err(|e| { @@ -978,6 +979,12 @@ async fn main() -> anyhow::Result<()> { metrics::gauge!("buzz_db_replica_fence_open").set(0.0); } } + // Probe liveness, ungated by staleness: how long since + // the probe last committed a heartbeat token. + if let Some(age) = pool_state.db.fence().heartbeat_age() { + metrics::gauge!("buzz_db_replica_heartbeat_age_seconds") + .set(age.as_secs_f64()); + } } let rs = pool_state.redis_pool.status(); From 5f81b10e5bd543384b3c3f12653d6221602736ed Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Mon, 27 Jul 2026 23:06:59 -0400 Subject: [PATCH 4/9] feat(replica): anchor routed reads in one REPEATABLE READ read-only transaction Wren's review hardening on 17ea2ff6a: the routed request previously ran proof + page + participants + aux as sequential autocommit statements on one held reader connection (the Rev 2 plan shape, sound via standby snapshot monotonicity). This strengthens the binding from monotone to exact and makes it robust against any future statement-level multiplexing between relay and reader: - proved_reader now opens the request with BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY (the strongest isolation a hot standby supports) and observes the heartbeat as the transaction's FIRST statement, so the snapshot the proof was taken against is exactly the snapshot every follow-up statement reads from. Begin failure fails closed to the writer like every other proof failure. - RouteDecision/ReadSession carry the open transaction; the page, participants batch, and the bridge aux closure all execute inside it. Dropping the ReadSession rolls the read-only transaction back and returns the connection to the pool. - New Postgres-gated regression test routed_request_holds_one_snapshot_across_page_and_aux distinguishes the transaction contract from mere connection reuse: a mid-request commit on the replica is visible to a control autocommit session but must NOT be visible to the held request session. Mutation-verified: swapping the BEGIN to a plain (READ COMMITTED) transaction makes the test fail on exactly the leak assertion. Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- crates/buzz-db/src/lib.rs | 175 +++++++++++++++++++++++----- crates/buzz-db/src/replica_fence.rs | 10 +- crates/buzz-relay/src/api/bridge.rs | 11 +- 3 files changed, 157 insertions(+), 39 deletions(-) diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index f5a67ed9a0..18485fcfc0 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -198,8 +198,15 @@ pub struct Db { /// The session that served (or will serve) a routed read, so follow-up /// queries in the same request (the channel-window aux closure) run on the -/// **same proved connection** — a different pooled reader session may sit at -/// a different replay position, so hopping sessions would discard the proof. +/// **same proved snapshot** — a different pooled reader session may sit at a +/// different replay position, and even the same connection advances its +/// snapshot between autocommit statements. +/// +/// `Replica` holds the request's `REPEATABLE READ, READ ONLY` transaction: +/// the heartbeat observation was its first statement, so the snapshot the +/// proof was taken against is exactly the snapshot every follow-up sees. +/// Dropping the session rolls the read-only transaction back and returns +/// the connection to the pool. /// /// `Writer` carries the writer pool: follow-ups there are authoritative by /// construction and need no session pinning. @@ -208,8 +215,8 @@ pub struct ReadSession { } enum ReadSessionInner { - /// A replica session whose heartbeat observation proved fence coverage. - Replica(sqlx::pool::PoolConnection), + /// The proved replica request transaction (snapshot-anchored). + Replica(sqlx::Transaction<'static, sqlx::Postgres>), /// The writer pool (cheap clone; Arc-backed). Writer(PgPool), } @@ -218,7 +225,7 @@ impl ReadSession { /// Query events on this session (see [`Db::query_events`]). pub async fn query_events(&mut self, q: &EventQuery) -> Result> { match &mut self.inner { - ReadSessionInner::Replica(conn) => event::query_events_on(conn, q).await, + ReadSessionInner::Replica(tx) => event::query_events_on(&mut *tx, q).await, ReadSessionInner::Writer(pool) => event::query_events(pool, q).await, } } @@ -231,9 +238,10 @@ impl ReadSession { /// Where one routed read is served (see [`Db::route_read`]). enum RouteDecision { - /// A reader session that proved this fence entry — the page runs here. + /// A reader request transaction whose first-statement heartbeat + /// observation proved this fence entry — the page runs inside it. Replica( - sqlx::pool::PoolConnection, + sqlx::Transaction<'static, sqlx::Postgres>, replica_fence::TokenEntry, ), /// Fail closed: serve from the writer pool. @@ -580,35 +588,43 @@ impl Db { self.read_pool.is_some() } - /// Acquire a reader session and complete the connection-local half of - /// the fence proof: observe the heartbeat token/epoch **on that exact - /// session** and resolve it against the retained ring. Returns the - /// proved session together with the strongest [`replica_fence::TokenEntry`] - /// its observation supports, or the fail-closed reason for route - /// metrics. + /// Open a reader request transaction and complete the connection-local + /// half of the fence proof: `BEGIN ISOLATION LEVEL REPEATABLE READ, READ + /// ONLY`, then observe the heartbeat token/epoch as the transaction's + /// **first statement** — anchoring the snapshot every follow-up + /// statement (page, participants, aux closure) sees to exactly the + /// snapshot the proof was taken against — and resolve it against the + /// retained ring. Returns the open transaction together with the + /// strongest [`replica_fence::TokenEntry`] its observation supports, or + /// the fail-closed reason for route metrics. /// - /// Everything but `Ok` fails closed — acquire failure, missing - /// heartbeat row (migration not yet replayed there), observation error, - /// epoch mismatch, or a token below every retained entry all route the - /// request to the writer. + /// `REPEATABLE READ` is the strongest isolation a hot standby supports + /// (`SERIALIZABLE` is writer-only); `READ ONLY` documents intent and + /// rejects accidental writes. Everything but `Ok` fails closed — begin + /// failure, missing heartbeat row (migration not yet replayed there), + /// observation error, epoch mismatch, or a token below every retained + /// entry all route the request to the writer. async fn proved_reader( &self, read_pool: &PgPool, ) -> std::result::Result< ( - sqlx::pool::PoolConnection, + sqlx::Transaction<'static, sqlx::Postgres>, replica_fence::TokenEntry, ), &'static str, > { - let mut conn = match read_pool.acquire().await { - Ok(conn) => conn, + let mut tx = match read_pool + .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .await + { + Ok(tx) => tx, Err(e) => { - tracing::warn!(error = %e, "reader acquire failed; routing to writer"); + tracing::warn!(error = %e, "reader transaction begin failed; routing to writer"); return Err("reader_validation_error"); } }; - let obs = match replica_fence::observe_heartbeat(&mut conn).await { + let obs = match replica_fence::observe_heartbeat(&mut tx).await { Ok(Some(observation)) => observation, Ok(None) => return Err("reader_validation_error"), Err(e) => { @@ -622,9 +638,9 @@ impl Db { token = obs.token, proved_token = entry.token, backend = %obs.backend, - "reader session proved fence coverage" + "reader snapshot proved fence coverage" ); - Ok((conn, entry)) + Ok((tx, entry)) } replica_fence::ResolveOutcome::EpochMismatch => Err("reader_validation_error"), replica_fence::ResolveOutcome::TokenBehind => Err("reader_token_behind"), @@ -2186,9 +2202,9 @@ impl Db { Some(_) => ("thread_cursor", RoutePredicate::Cursor(None)), None => ("thread_head", RoutePredicate::Head), }; - if let RouteDecision::Replica(mut conn, entry) = self.route_read(path, predicate).await { + if let RouteDecision::Replica(mut tx, entry) = self.route_read(path, predicate).await { let replies = thread::get_thread_replies_on( - &mut conn, + &mut tx, community_id, root_event_id, depth_limit, @@ -2287,9 +2303,9 @@ impl Db { .route_read(path, RoutePredicate::from_channel_cursor(&cursor)) .await { - RouteDecision::Replica(mut conn, _entry) => { + RouteDecision::Replica(mut tx, _entry) => { let window = thread::get_channel_window_on( - &mut conn, + &mut tx, community_id, channel_id, limit, @@ -2300,7 +2316,7 @@ impl Db { Ok(( window, ReadSession { - inner: ReadSessionInner::Replica(conn), + inner: ReadSessionInner::Replica(tx), }, )) } @@ -2363,7 +2379,7 @@ impl Db { }, } match self.proved_reader(read_pool).await { - Ok((conn, entry)) => { + Ok((tx, entry)) => { let (holds, reason): (bool, &'static str) = match &predicate { RoutePredicate::Cursor(Some(ts)) => (*ts <= entry.fence_wall, "covered"), // No upper bound: the caller post-verifies the served @@ -2377,7 +2393,7 @@ impl Db { }; if holds { Self::record_route(path, "replica", reason); - RouteDecision::Replica(conn, entry) + RouteDecision::Replica(tx, entry) } else { // The session proves an older entry than the predicate // needs (replication lag) — fail closed. @@ -5848,6 +5864,103 @@ mod tests { drop_scratch_db(&admin, writer, &wname).await; } + /// Snapshot continuity (Wren, review of 17ea2ff6a): the routed request + /// runs inside ONE `REPEATABLE READ, READ ONLY` transaction whose first + /// statement was the heartbeat observation — so a row committed on the + /// replica *after* the proof must be invisible to every follow-up + /// statement in the same request (page, participants, aux). This + /// distinguishes the transaction contract from mere connection reuse: + /// autocommit statements on the same backend advance their snapshot + /// per statement and WOULD see the mid-request row. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn routed_request_holds_one_snapshot_across_page_and_aux() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (writer, wname) = create_scratch_db(&admin, "snap_w").await; + let (replica, rname) = create_scratch_db(&admin, "snap_r").await; + + let author = nostr::Keys::generate(); + let community = Uuid::new_v4(); + let channel = Uuid::new_v4(); + seed_community_channel(&writer, community, channel, &author).await; + seed_community_channel(&replica, community, channel, &author).await; + + let base = 1_700_000_000u64; + let m1 = signed_event_at(&author, "m1", base); + let m2 = signed_event_at(&author, "m2", base + 10); + for pool in [&writer, &replica] { + for ev in [&m1, &m2] { + insert_top_level(pool, community, channel, ev).await; + } + } + + let db = Db::from_pools(writer.clone(), replica.clone()); + db.fence().force_open_for_tests(chrono::Utc::now()); + let cid = CommunityId::from_uuid(community); + + // Head page on the writer yields the cursor for a replica-routed page. + let head = db + .get_channel_window(cid, channel, 1, None, None) + .await + .expect("head window"); + let cursor = head.next_cursor.expect("has_more implies next_cursor"); + + // Route the cursor page to the replica and HOLD the session. + let (window, mut session) = db + .get_channel_window_with_session(cid, channel, 10, Some(cursor), None) + .await + .expect("routed cursor window"); + assert!( + session.is_replica(), + "fixture must route this page to the replica" + ); + assert_eq!(window.rows.len(), 1, "page after m2 is [m1]"); + + // Mid-request: a new event commits on the replica (stands in for + // replay advancing between the page and the aux closure). + let mid = signed_event_at(&author, "mid-request-commit", base + 5); + insert_top_level(&replica, community, channel, &mid).await; + + // A fresh autocommit statement on ANOTHER session sees it — the row + // is really there (control for the assertion below). + let mut control = EventQuery::for_community(cid); + control.channel_id = Some(channel); + let visible_elsewhere = event::query_events(&replica, &control) + .await + .expect("control query"); + assert!( + visible_elsewhere + .iter() + .any(|se| se.event.content == "mid-request-commit"), + "control: the mid-request row must be committed and visible to a new snapshot" + ); + + // The held request session must NOT see it: its snapshot was + // anchored by the heartbeat observation, before the commit. + let mut aux = EventQuery::for_community(cid); + aux.channel_id = Some(channel); + let in_request = session.query_events(&aux).await.expect("aux query"); + assert!( + !in_request + .iter() + .any(|se| se.event.content == "mid-request-commit"), + "request transaction must hold the proof-time snapshot; a \ + mid-request commit leaking in means the aux ran outside the \ + request transaction (autocommit connection reuse)" + ); + // Rows from the proof-time snapshot are still served. + assert!( + in_request.iter().any(|se| se.event.content == "m1"), + "proof-time rows must remain visible in the request snapshot" + ); + + drop(session); + drop_scratch_db(&admin, replica, &rname).await; + drop_scratch_db(&admin, writer, &wname).await; + } + /// Head gate (Predicate A): with the budget unset, a head fetch reads /// the writer even over an open fence; with a budget set and a fresh /// proved entry, the head page is served by the replica session diff --git a/crates/buzz-db/src/replica_fence.rs b/crates/buzz-db/src/replica_fence.rs index b56c359115..68fcc04203 100644 --- a/crates/buzz-db/src/replica_fence.rs +++ b/crates/buzz-db/src/replica_fence.rs @@ -33,9 +33,13 @@ //! //! Unlike the previous WAL-LSN observation (`pg_last_wal_replay_lsn()`, which //! Aurora reader endpoints hide), the token observation is portable and — -//! critically — **connection-local**: the proof binds to the exact reader -//! session that will serve the page, never to a different pooled session -//! (readers behind one endpoint may sit at different replay positions). +//! critically — **snapshot-local**: routing opens a `REPEATABLE READ, READ +//! ONLY` transaction on the reader session that will serve the page and +//! observes the heartbeat as its first statement, so the proof binds to the +//! exact snapshot every follow-up statement in the request (page, +//! participants, aux closure) reads from — never to a different pooled +//! session (readers behind one endpoint may sit at different replay +//! positions), and never to a later autocommit snapshot on the same wire. //! An observed token lower than the newest retained `M` is ordinary //! replication lag, not a fault; the resolver simply proves from an older //! retained `M`. Regression detection is writer-side only: a non-monotonic diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 6cbd31202f..470332d786 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -485,11 +485,12 @@ async fn handle_channel_window_filter( // 2. Aux closure: reactions/deletions/edits targeting retained rows, plus // deletions targeting those aux events (the transitive second hop). - // One round trip for the client instead of an #e fan-out. Runs on the - // SAME session that served the window: when the page came from a - // proved replica connection, hopping to another pooled session (which - // may sit at an older replay position) could drop aux rows the served - // page's proof already covers. + // One round trip for the client instead of an #e fan-out. Runs in the + // SAME request transaction that served the window: when the page came + // from a proved replica session, the heartbeat observation anchored a + // REPEATABLE READ snapshot, so the aux hops see exactly the state the + // proof covered — another pooled session (or even another autocommit + // statement) could sit at a different replay position. if extension_flag(raw, "include_aux") && !row_ids_hex.is_empty() { let mut seen_aux: std::collections::HashSet = std::collections::HashSet::new(); From 221245ab90f00cfe52fd8976adb34a675817d156 Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Tue, 28 Jul 2026 00:08:47 -0400 Subject: [PATCH 5/9] feat(replica): enriched backend identity tuple for route evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Max's E2E finding #2: inet_server_addr() alone is weaker than the plan's production identity tuple and cannot distinguish readers behind one address (or record which backend process served the proof). Blocked the Aurora canary on this fast-follow. - observe_heartbeat now reports `addr:port pid=N` (`local pid=N` on unix sockets), and prefixes the Aurora instance id (`aurora_server_id() @ ...`) when the endpoint supports it. - aurora_server_id() cannot be referenced parse-safely on plain Postgres, so support is probed once per process by reader_supports_aurora_identity on a plain autocommit checkout — never inside the request transaction, where undefined_function would abort it. SQLSTATE 42883 caches a definitive false; transient errors degrade to the plain tuple for that request and retry later. Identity is evidence, never a routing gate. - New Postgres-gated test: the capability probe answers false (not an error) on plain Postgres and leaves the session usable; the existing probe test now pins the addr:port/pid shape. Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- crates/buzz-db/src/lib.rs | 35 ++++++++++- crates/buzz-db/src/replica_fence.rs | 90 ++++++++++++++++++++++++----- 2 files changed, 111 insertions(+), 14 deletions(-) diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 18485fcfc0..a4a6f3b688 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -194,6 +194,12 @@ pub struct Db { /// default) — bounded-stale head semantics are a product decision, not /// an invariant, so the gate ships off. pub(crate) replica_head_max_age: Option, + /// Whether the reader endpoint supports `aurora_server_id()` — probed + /// once per process on the first routed read (on a plain autocommit + /// checkout, outside any request transaction) and cached. Unset means + /// not yet probed (or the probe hit a transient error and will retry). + /// Shared across `Db` clones. + pub(crate) reader_aurora_identity: std::sync::Arc>, } /// The session that served (or will serve) a routed read, so follow-up @@ -473,6 +479,7 @@ impl Db { read_pool, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), replica_head_max_age, + reader_aurora_identity: std::sync::Arc::new(std::sync::OnceLock::new()), }) } @@ -512,6 +519,7 @@ impl Db { read_pool: None, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), replica_head_max_age: None, + reader_aurora_identity: std::sync::Arc::new(std::sync::OnceLock::new()), } } @@ -529,6 +537,7 @@ impl Db { read_pool: Some(read_pool), fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), replica_head_max_age: None, + reader_aurora_identity: std::sync::Arc::new(std::sync::OnceLock::new()), } } @@ -614,6 +623,7 @@ impl Db { ), &'static str, > { + let aurora = self.reader_aurora_capability(read_pool).await; let mut tx = match read_pool .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY") .await @@ -624,7 +634,7 @@ impl Db { return Err("reader_validation_error"); } }; - let obs = match replica_fence::observe_heartbeat(&mut tx).await { + let obs = match replica_fence::observe_heartbeat(&mut tx, aurora).await { Ok(Some(observation)) => observation, Ok(None) => return Err("reader_validation_error"), Err(e) => { @@ -647,6 +657,29 @@ impl Db { } } + /// Whether the reader endpoint supports `aurora_server_id()`, probed + /// once per process and cached (see [`Db::reader_aurora_identity`]). + /// The probe runs on a plain autocommit checkout — never inside the + /// request transaction, where an undefined-function error would abort + /// it. Probe failure (acquire or transient) degrades to the plain + /// identity tuple for THIS request without caching, so a later request + /// retries; identity is evidence, never a routing gate. + async fn reader_aurora_capability(&self, read_pool: &PgPool) -> bool { + if let Some(cached) = self.reader_aurora_identity.get() { + return *cached; + } + let Ok(mut conn) = read_pool.acquire().await else { + return false; + }; + match replica_fence::reader_supports_aurora_identity(&mut conn).await { + Ok(supported) => *self.reader_aurora_identity.get_or_init(|| supported), + Err(e) => { + tracing::debug!(error = %e, "aurora identity probe failed; will retry"); + false + } + } + } + /// Record one route decision (Rev 2 observability): which path, where it /// went, and why. fn record_route(path: &'static str, decision: &'static str, reason: &'static str) { diff --git a/crates/buzz-db/src/replica_fence.rs b/crates/buzz-db/src/replica_fence.rs index 68fcc04203..11dccd575e 100644 --- a/crates/buzz-db/src/replica_fence.rs +++ b/crates/buzz-db/src/replica_fence.rs @@ -672,20 +672,52 @@ pub async fn probe_once(writer: &PgPool, fence: &ReplicaFence) -> Result Result { + match sqlx::query("SELECT aurora_server_id()") + .fetch_one(&mut *conn) + .await + { + Ok(_) => Ok(true), + Err(sqlx::Error::Database(e)) if e.code().as_deref() == Some("42883") => Ok(false), + Err(e) => Err(e), + } +} + /// Observe the heartbeat on a specific reader session — the /// connection-local half of the proof. Returns the observed token/epoch -/// plus the backend identity of the session (`inet_server_addr()`; -/// `"local"` on unix sockets) for route-decision evidence. `None` when the -/// row is missing there (migration not yet replayed): fail closed. +/// plus the backend identity of the session for route-decision evidence: +/// `addr:port pid=N` (`local` on unix sockets), prefixed with the Aurora +/// instance id when `aurora` is set (only pass `true` after +/// [`reader_supports_aurora_identity`] confirmed it — the function +/// reference fails at parse time on plain Postgres). `None` when the row +/// is missing there (migration not yet replayed): fail closed. pub async fn observe_heartbeat( conn: &mut PgConnection, + aurora: bool, ) -> Result, sqlx::Error> { - let row = sqlx::query( - "SELECT token, epoch, COALESCE(host(inet_server_addr()), 'local') AS backend \ - FROM replica_heartbeat WHERE id = 1", - ) - .fetch_optional(&mut *conn) - .await?; + const ADDR_PID: &str = "COALESCE(host(inet_server_addr()) || ':' || \ + inet_server_port()::text, 'local') || ' pid=' || pg_backend_pid()::text"; + let sql = if aurora { + format!( + "SELECT token, epoch, aurora_server_id() || ' @ ' || {ADDR_PID} AS backend \ + FROM replica_heartbeat WHERE id = 1" + ) + } else { + format!( + "SELECT token, epoch, {ADDR_PID} AS backend \ + FROM replica_heartbeat WHERE id = 1" + ) + }; + let row = sqlx::query(sqlx::AssertSqlSafe(sql)) + .fetch_optional(&mut *conn) + .await?; Ok(row.map(|r| HeartbeatObservation { token: r.get("token"), epoch: r.get("epoch"), @@ -996,6 +1028,28 @@ mod tests { .expect("drop role"); } + /// The Aurora identity capability probe must answer a definitive + /// `false` on plain Postgres (undefined_function), not error — and the + /// error path must not poison the connection for later statements. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn aurora_identity_probe_reports_false_on_plain_postgres() { + let pool = PgPool::connect(&test_db_url()).await.expect("connect"); + let mut conn = pool.acquire().await.expect("conn"); + assert!( + !reader_supports_aurora_identity(&mut conn) + .await + .expect("probe must not error on plain postgres"), + "plain postgres must report no aurora identity support" + ); + // The failed function lookup must not have wedged the session. + let one: i32 = sqlx::query_scalar("SELECT 1") + .fetch_one(&mut *conn) + .await + .expect("connection usable after probe"); + assert_eq!(one, 1); + } + /// End-to-end probe against a real database: each probe commits a /// strictly newer token, records a retained entry, and a session on the /// same database observes a token/epoch that resolves that entry. @@ -1018,12 +1072,22 @@ mod tests { // A "reader" session on the same database observes at least the // second token and proves the newest retained entry. let mut conn = pool.acquire().await.expect("reader conn"); - let obs = observe_heartbeat(&mut conn) + let obs = observe_heartbeat(&mut conn, false) .await .expect("observe") .expect("heartbeat row present"); assert!(obs.token >= second.token); - assert!(!obs.backend.is_empty(), "backend identity recorded"); + assert!( + obs.backend.contains(" pid="), + "backend identity must carry the backend pid, got {:?}", + obs.backend + ); + // TCP fixtures also carry addr:port; unix-socket fixtures read 'local'. + assert!( + obs.backend.starts_with("local pid=") || obs.backend.contains(':'), + "backend identity must carry addr:port or 'local', got {:?}", + obs.backend + ); let proof = fence .resolve(obs.token, obs.epoch) .proved() @@ -1052,7 +1116,7 @@ mod tests { let before = probe_once(&pool, &fence).await.expect("probe"); let mut conn = pool.acquire().await.expect("conn"); - let old_epoch = observe_heartbeat(&mut conn) + let old_epoch = observe_heartbeat(&mut conn, false) .await .expect("observe") .expect("row") @@ -1072,7 +1136,7 @@ mod tests { "old-epoch observations must fail closed after rotation" ); // A fresh observation on the new timeline proves the rotated entry. - let obs = observe_heartbeat(&mut conn) + let obs = observe_heartbeat(&mut conn, false) .await .expect("observe") .expect("row"); From fedb463689291995d0ba3e96604fd2879d5a008b Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Tue, 28 Jul 2026 00:29:02 -0400 Subject: [PATCH 6/9] fix(replica): use the Aurora PostgreSQL identity function name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aurora_server_id is the MySQL-family spelling; Aurora PostgreSQL exposes aurora_db_instance_identifier() (AWS Aurora PostgreSQL user guide; awslabs/pg-collector). As written, the capability probe would have hit 42883 on real Aurora, cached a permanent false, and silently stripped the instance id from canary evidence — the exact gap the fast-follow exists to close (Wren, delta review of a472327). The function name now lives in one const (AURORA_IDENTITY_FN) shared by the probe and the observation query, pinned by a unit test so the near-miss cannot recur. The real Aurora canary proves the positive branch. Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- crates/buzz-db/src/lib.rs | 6 ++++-- crates/buzz-db/src/replica_fence.rs | 31 ++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index a4a6f3b688..5893ee81ec 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -194,7 +194,8 @@ pub struct Db { /// default) — bounded-stale head semantics are a product decision, not /// an invariant, so the gate ships off. pub(crate) replica_head_max_age: Option, - /// Whether the reader endpoint supports `aurora_server_id()` — probed + /// Whether the reader endpoint supports the Aurora PostgreSQL identity + /// function ([`replica_fence::AURORA_IDENTITY_FN`]) — probed /// once per process on the first routed read (on a plain autocommit /// checkout, outside any request transaction) and cached. Unset means /// not yet probed (or the probe hit a transient error and will retry). @@ -657,7 +658,8 @@ impl Db { } } - /// Whether the reader endpoint supports `aurora_server_id()`, probed + /// Whether the reader endpoint supports the Aurora PostgreSQL identity + /// function ([`replica_fence::AURORA_IDENTITY_FN`]), probed /// once per process and cached (see [`Db::reader_aurora_identity`]). /// The probe runs on a plain autocommit checkout — never inside the /// request transaction, where an undefined-function error would abort diff --git a/crates/buzz-db/src/replica_fence.rs b/crates/buzz-db/src/replica_fence.rs index 11dccd575e..08f8498675 100644 --- a/crates/buzz-db/src/replica_fence.rs +++ b/crates/buzz-db/src/replica_fence.rs @@ -672,7 +672,15 @@ pub async fn probe_once(writer: &PgPool, fence: &ReplicaFence) -> Result Result Result { - match sqlx::query("SELECT aurora_server_id()") - .fetch_one(&mut *conn) - .await + match sqlx::query(sqlx::AssertSqlSafe(format!( + "SELECT {AURORA_IDENTITY_FN}()" + ))) + .fetch_one(&mut *conn) + .await { Ok(_) => Ok(true), Err(sqlx::Error::Database(e)) if e.code().as_deref() == Some("42883") => Ok(false), @@ -706,7 +716,7 @@ pub async fn observe_heartbeat( inet_server_port()::text, 'local') || ' pid=' || pg_backend_pid()::text"; let sql = if aurora { format!( - "SELECT token, epoch, aurora_server_id() || ' @ ' || {ADDR_PID} AS backend \ + "SELECT token, epoch, {AURORA_IDENTITY_FN}() || ' @ ' || {ADDR_PID} AS backend \ FROM replica_heartbeat WHERE id = 1" ) } else { @@ -1028,6 +1038,17 @@ mod tests { .expect("drop role"); } + /// The Aurora PostgreSQL identity function name is exact — the + /// MySQL-family near-miss (`aurora_server_id`) would make the + /// capability probe cache a permanent false on real Aurora and + /// silently strip the instance id from canary evidence (Wren, delta + /// review of a472327). AWS reference: aurora_db_instance_identifier() + /// (Aurora PostgreSQL user guide; also awslabs/pg-collector). + #[test] + fn aurora_identity_function_name_is_the_postgres_one() { + assert_eq!(AURORA_IDENTITY_FN, "aurora_db_instance_identifier"); + } + /// The Aurora identity capability probe must answer a definitive /// `false` on plain Postgres (undefined_function), not error — and the /// error path must not poison the connection for later statements. From 7d9f54c1ff6bd5d271b876d6a160bfb310b67947 Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Tue, 28 Jul 2026 12:22:33 -0400 Subject: [PATCH 7/9] Fail closed to the writer on mid-request replica failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dawn's first-principles review of 1b0aa0dfa found the one path where the "degraded capacity, never holes" invariant did not hold: a replica-routed query that errors AFTER the fence proof (the live shape is a hot-standby recovery conflict — 40001/25P02 — cancelling the held REPEATABLE READ snapshot under max_standby_streaming_delay) propagated the error to the bridge as an HTTP 500 instead of re-running on the writer. - get_channel_window_with_session: a replica-arm query error logs, records route(path, writer, replica_error), drops the reader transaction, and re-runs the page on the writer. - get_thread_replies: same fallback for the thread replica arm. - ReadSession::query_events: a mid-request failure of the held replica transaction (page succeeded, aux hop failed) re-runs the query on the writer and permanently degrades the session; counted by the new buzz_db_read_session_degraded counter, deliberately NOT a buzz_db_route_decision event so offload stays one event per request. Metrics correctness (Dawn's second finding): the replica route decision is now recorded only when the page is actually served from the replica. Previously a thread cursor page that failed post-verification emitted BOTH thread_cursor/replica/covered and thread_eof/writer/stale, overcounting replica offload — the exact number the Aurora canary verdict reads. Db::read() is demoted to #[cfg(test)]: it had zero production callers and handed out the replica pool with no fence proof — the bug class this machinery exists to eliminate. The head-gate doc no longer claims the WS since-overlap union covers fresh events (that client change has not shipped); it now says so explicitly and forbids enabling the budget until it has. Three new PG-gated regression tests: window fallback, thread fallback, and held-session degradation via pg_terminate_backend, each guarded against vacuous passes by proving replica routing while healthy. Reviewed-by findings: Dawn (first-principles review, 2026-07-28) Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- crates/buzz-db/src/lib.rs | 443 ++++++++++++++++++++++++++++++++------ 1 file changed, 383 insertions(+), 60 deletions(-) diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 5893ee81ec..f11ed3939c 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -222,36 +222,75 @@ pub struct ReadSession { } enum ReadSessionInner { - /// The proved replica request transaction (snapshot-anchored). - Replica(sqlx::Transaction<'static, sqlx::Postgres>), + /// The proved replica request transaction (snapshot-anchored), plus the + /// writer pool so a mid-request replica failure (e.g. a hot-standby + /// recovery conflict cancelling the held snapshot) degrades the session + /// to the writer instead of surfacing an error: degraded capacity, + /// never holes — and never a 500 the writer could have served. + Replica { + tx: sqlx::Transaction<'static, sqlx::Postgres>, + writer: PgPool, + }, /// The writer pool (cheap clone; Arc-backed). Writer(PgPool), } impl ReadSession { /// Query events on this session (see [`Db::query_events`]). + /// + /// If the proved replica transaction fails mid-request, the session + /// permanently degrades to the writer and the query is re-run there. + /// The writer is always at or ahead of any replica replay position, so + /// the degraded follow-up can only observe *more* than the proof-time + /// snapshot, never less — fresher aux rows, the same failure semantics + /// as a request that routed to the writer to begin with. pub async fn query_events(&mut self, q: &EventQuery) -> Result> { - match &mut self.inner { - ReadSessionInner::Replica(tx) => event::query_events_on(&mut *tx, q).await, - ReadSessionInner::Writer(pool) => event::query_events(pool, q).await, - } + let degraded = match &mut self.inner { + ReadSessionInner::Replica { tx, writer } => { + match event::query_events_on(tx, q).await { + Ok(rows) => return Ok(rows), + Err(e) => { + tracing::warn!( + error = %e, + "replica session query failed mid-request; degrading to writer" + ); + // Deliberately not a `buzz_db_route_decision` event: + // the page's route was already recorded, and the + // offload metric must stay one-event-per-request. + metrics::counter!("buzz_db_read_session_degraded").increment(1); + writer.clone() + } + } + } + ReadSessionInner::Writer(pool) => return event::query_events(pool, q).await, + }; + // Replacing the inner drops the replica transaction (rolling it + // back and returning the reader connection to its pool). + self.inner = ReadSessionInner::Writer(degraded.clone()); + event::query_events(°raded, q).await } /// Whether this session is a proved replica connection (observability). pub fn is_replica(&self) -> bool { - matches!(self.inner, ReadSessionInner::Replica(_)) + matches!(self.inner, ReadSessionInner::Replica { .. }) } } /// Where one routed read is served (see [`Db::route_read`]). enum RouteDecision { /// A reader request transaction whose first-statement heartbeat - /// observation proved this fence entry — the page runs inside it. + /// observation proved this fence entry — the page runs inside it. The + /// `&'static str` is the metric reason (`covered`/`fresh`); the caller + /// records the route only once the page is actually served from the + /// replica, so a post-verification writer re-run or a mid-query replica + /// failure emits exactly one `buzz_db_route_decision` event per request + /// (the offload percentage is read straight off `decision="replica"`). Replica( sqlx::Transaction<'static, sqlx::Postgres>, replica_fence::TokenEntry, + &'static str, ), - /// Fail closed: serve from the writer pool. + /// Fail closed: serve from the writer pool (already recorded). Writer, } @@ -585,11 +624,13 @@ impl Db { /// The pool for lag-tolerant reads: the read replica when configured, /// otherwise the writer pool. /// - /// Routing contract — a query may use this pool only when a stale (bounded - /// replication lag) result is acceptable to its caller. Keyset-cursor - /// pagination over immutable history qualifies; head-of-channel fetches, - /// auth/membership checks, locks, and anything inside a transaction do not. - pub fn read(&self) -> &PgPool { + /// Removed as a public escape hatch (Dawn, review of 1b0aa0dfa): the + /// raw replica pool carries **no fence proof**, which is exactly the + /// bug class the routed-read machinery exists to eliminate. All replica + /// reads must go through [`Db::route_read`]-backed entry points; this + /// remains only for the fence's own plumbing tests. + #[cfg(test)] + fn read(&self) -> &PgPool { self.read_pool.as_ref().unwrap_or(&self.pool) } @@ -2237,8 +2278,10 @@ impl Db { Some(_) => ("thread_cursor", RoutePredicate::Cursor(None)), None => ("thread_head", RoutePredicate::Head), }; - if let RouteDecision::Replica(mut tx, entry) = self.route_read(path, predicate).await { - let replies = thread::get_thread_replies_on( + if let RouteDecision::Replica(mut tx, entry, reason) = + self.route_read(path, predicate).await + { + match thread::get_thread_replies_on( &mut tx, community_id, root_event_id, @@ -2246,21 +2289,39 @@ impl Db { limit, cursor, ) - .await?; - if cursor.is_none() { - // Predicate A: bounded-stale head page, served as proved. - return Ok(replies); - } - let full = replies.len() >= limit as usize; - let below_fence = replies - .last() - .is_some_and(|tail| tail.created_at <= entry.fence_wall); - if full && below_fence { - return Ok(replies); + .await + { + Ok(replies) => { + if cursor.is_none() { + // Predicate A: bounded-stale head page, served as proved. + Self::record_route(path, "replica", reason); + return Ok(replies); + } + let full = replies.len() >= limit as usize; + let below_fence = replies + .last() + .is_some_and(|tail| tail.created_at <= entry.fence_wall); + if full && below_fence { + Self::record_route(path, "replica", reason); + return Ok(replies); + } + // Candidate terminal page, or page reaching above the + // proved wall — verify against the writer. Recorded as + // the request's ONLY route event: the replica leg was + // discarded, so counting it would overstate offload. + Self::record_route("thread_eof", "writer", "stale"); + } + Err(e) => { + // Mid-request replica failure (e.g. a hot-standby + // recovery conflict) fails closed to the writer. + tracing::warn!( + error = %e, + path, + "replica thread query failed; re-running on writer" + ); + Self::record_route(path, "writer", "replica_error"); + } } - // Candidate terminal page, or page reaching above the proved - // wall — verify against the writer. - Self::record_route("thread_eof", "writer", "stale"); } thread::get_thread_replies( &self.pool, @@ -2317,7 +2378,11 @@ impl Db { /// ([`DbConfig::replica_head_max_age_secs`], default off) and the /// proved entry is within the budget. This trades a bounded staleness /// window (budget plus probe cadence) on the GET leg for writer - /// offload; the WS `since`-overlap union covers fresh events. + /// offload. NOTE: enabling the budget also breaks read-your-own-writes + /// on the GET leg; the client-side WS `since`-overlap union intended + /// to cover fresh events has NOT shipped yet — do not enable + /// `BUZZ_REPLICA_HEAD_MAX_AGE_SECS` until it has, proven by a + /// post-then-immediately-refetch test. /// /// Every failure fails closed to the writer and is recorded in /// `buzz_db_route_decision`. @@ -2338,41 +2403,62 @@ impl Db { .route_read(path, RoutePredicate::from_channel_cursor(&cursor)) .await { - RouteDecision::Replica(mut tx, _entry) => { - let window = thread::get_channel_window_on( + RouteDecision::Replica(mut tx, _entry, reason) => { + match thread::get_channel_window_on( &mut tx, community_id, channel_id, limit, - cursor, + cursor.clone(), kind_filter, ) - .await?; - Ok(( - window, - ReadSession { - inner: ReadSessionInner::Replica(tx), - }, - )) - } - RouteDecision::Writer => { - let window = thread::get_channel_window( - &self.pool, - community_id, - channel_id, - limit, - cursor, - kind_filter, - ) - .await?; - Ok(( - window, - ReadSession { - inner: ReadSessionInner::Writer(self.pool.clone()), - }, - )) + .await + { + Ok(window) => { + Self::record_route(path, "replica", reason); + return Ok(( + window, + ReadSession { + inner: ReadSessionInner::Replica { + tx, + writer: self.pool.clone(), + }, + }, + )); + } + Err(e) => { + // A mid-request replica failure (e.g. a hot-standby + // recovery conflict cancelling the held snapshot) + // fails closed to the writer: a stale-but-served + // page, never an error the writer could have + // answered. Dropping `tx` rolls the reader + // transaction back. + tracing::warn!( + error = %e, + path, + "replica window query failed; re-running on writer" + ); + Self::record_route(path, "writer", "replica_error"); + } + } } + RouteDecision::Writer => {} } + let window = thread::get_channel_window( + &self.pool, + community_id, + channel_id, + limit, + cursor, + kind_filter, + ) + .await?; + Ok(( + window, + ReadSession { + inner: ReadSessionInner::Writer(self.pool.clone()), + }, + )) } /// Shared route decision for one read: evaluate the predicate against a @@ -2427,8 +2513,7 @@ impl Db { ), }; if holds { - Self::record_route(path, "replica", reason); - RouteDecision::Replica(tx, entry) + RouteDecision::Replica(tx, entry, reason) } else { // The session proves an older entry than the predicate // needs (replication lag) — fail closed. @@ -5899,6 +5984,244 @@ mod tests { drop_scratch_db(&admin, writer, &wname).await; } + /// Fail-closed on a mid-request replica failure (Dawn, review of + /// 1b0aa0dfa): a replica-routed page whose query errors *after* the + /// proof (the live shape is a hot-standby recovery conflict — 40001 / + /// 25P02 — cancelling the held snapshot under `max_standby_streaming_delay`) + /// must be re-run on the writer and served, never surfaced as an error + /// the writer could have answered. Degraded capacity, never holes. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn replica_window_failure_falls_back_to_writer() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (writer, wname) = create_scratch_db(&admin, "fb_w").await; + let (replica, rname) = create_scratch_db(&admin, "fb_r").await; + + let author = nostr::Keys::generate(); + let community = Uuid::new_v4(); + let channel = Uuid::new_v4(); + seed_community_channel(&writer, community, channel, &author).await; + seed_community_channel(&replica, community, channel, &author).await; + + let base = 1_700_000_000u64; + let m1 = signed_event_at(&author, "m1", base); + let m2 = signed_event_at(&author, "m2", base + 10); + let m3 = signed_event_at(&author, "m3", base + 20); + for pool in [&writer, &replica] { + for ev in [&m1, &m2, &m3] { + insert_top_level(pool, community, channel, ev).await; + } + } + let marker = signed_event_at(&author, "replica-only-marker", base + 5); + insert_top_level(&replica, community, channel, &marker).await; + + let db = Db::from_pools(writer.clone(), replica.clone()); + db.fence().force_open_for_tests(chrono::Utc::now()); + let cid = CommunityId::from_uuid(community); + + let head = db + .get_channel_window(cid, channel, 1, None, None) + .await + .expect("head window"); + let cursor = head.next_cursor.expect("has_more implies next_cursor"); + + // Guard against a vacuous pass: the cursor page must actually be + // replica-eligible before we break the replica. + let healthy = db + .get_channel_window(cid, channel, 10, Some(cursor.clone()), None) + .await + .expect("healthy cursor window"); + assert!( + healthy + .rows + .iter() + .any(|r| r.stored_event.event.content == "replica-only-marker"), + "fixture must route the cursor page to the replica while healthy" + ); + + // Break the replica AFTER the proof point: the heartbeat table stays + // intact (the observation succeeds), the page query then fails. + sqlx::query("DROP TABLE events CASCADE") + .execute(&replica) + .await + .expect("drop replica events"); + + let page = db + .get_channel_window(cid, channel, 10, Some(cursor), None) + .await + .expect("replica failure must fall back to the writer, not error"); + let contents: Vec<&str> = page + .rows + .iter() + .map(|r| r.stored_event.event.content.as_str()) + .collect(); + assert_eq!( + contents, + vec!["m2", "m1"], + "fallback page must be the writer's answer (no replica marker)" + ); + + drop_scratch_db(&admin, replica, &rname).await; + drop_scratch_db(&admin, writer, &wname).await; + } + + /// [`replica_window_failure_falls_back_to_writer`] for the thread-replies + /// path: a replica-routed thread page whose query errors after the proof + /// re-runs on the writer instead of surfacing an error. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn replica_thread_failure_falls_back_to_writer() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (writer, wname) = create_scratch_db(&admin, "fbt_w").await; + let (replica, rname) = create_scratch_db(&admin, "fbt_r").await; + + let author = nostr::Keys::generate(); + let community = Uuid::new_v4(); + let channel = Uuid::new_v4(); + seed_community_channel(&writer, community, channel, &author).await; + seed_community_channel(&replica, community, channel, &author).await; + + let base = 1_700_000_000u64; + let root = signed_event_at(&author, "root", base); + for pool in [&writer, &replica] { + insert_top_level(pool, community, channel, &root).await; + } + let replies: Vec = (1..=3) + .map(|i| signed_event_at(&author, &format!("r{i}"), base + 10 * i as u64)) + .collect(); + for pool in [&writer, &replica] { + for reply in &replies { + insert_thread_reply(pool, community, channel, &root, reply).await; + } + } + // Replica-only divergent reply between r2 and r3 marks replica serves. + let ghost = signed_event_at(&author, "replica-only-ghost", base + 25); + insert_thread_reply(&replica, community, channel, &root, &ghost).await; + + let db = Db::from_pools(writer.clone(), replica.clone()); + db.fence().force_open_for_tests(chrono::Utc::now()); + let cid = CommunityId::from_uuid(community); + + let page1 = db + .get_thread_replies(cid, root.id.as_bytes(), Some(10), 2, None) + .await + .expect("head page"); + let cur = thread_cursor(page1.last().expect("page 1 non-empty")); + + // Healthy: the full page after r2 is the replica's [ghost]. + let healthy = db + .get_thread_replies(cid, root.id.as_bytes(), Some(10), 1, Some(&cur)) + .await + .expect("healthy replica page"); + assert_eq!( + healthy[0].stored_event.event.content, "replica-only-ghost", + "fixture must route the cursor page to the replica while healthy" + ); + + sqlx::query("DROP TABLE events CASCADE") + .execute(&replica) + .await + .expect("drop replica events"); + + let page = db + .get_thread_replies(cid, root.id.as_bytes(), Some(10), 1, Some(&cur)) + .await + .expect("replica failure must fall back to the writer, not error"); + assert_eq!( + page[0].stored_event.event.content, "r3", + "fallback page must be the writer's answer" + ); + + drop_scratch_db(&admin, replica, &rname).await; + drop_scratch_db(&admin, writer, &wname).await; + } + + /// Mid-request degradation of the held session (Dawn, review of + /// 1b0aa0dfa): when the proved replica transaction dies between the page + /// and an aux follow-up (stand-in: `pg_terminate_backend` on the reader + /// connection, the same tx-fatal shape as a recovery-conflict cancel), + /// [`ReadSession::query_events`] must re-run the query on the writer and + /// permanently degrade the session instead of surfacing the error. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn read_session_degrades_to_writer_when_replica_connection_dies() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (writer, wname) = create_scratch_db(&admin, "deg_w").await; + let (replica, rname) = create_scratch_db(&admin, "deg_r").await; + + let author = nostr::Keys::generate(); + let community = Uuid::new_v4(); + let channel = Uuid::new_v4(); + seed_community_channel(&writer, community, channel, &author).await; + seed_community_channel(&replica, community, channel, &author).await; + + let base = 1_700_000_000u64; + let m1 = signed_event_at(&author, "m1", base); + let m2 = signed_event_at(&author, "m2", base + 10); + for pool in [&writer, &replica] { + for ev in [&m1, &m2] { + insert_top_level(pool, community, channel, ev).await; + } + } + // Writer-only row proves the degraded aux ran on the writer. + let fresh = signed_event_at(&author, "fresh-writer-only", base + 20); + insert_top_level(&writer, community, channel, &fresh).await; + + let db = Db::from_pools(writer.clone(), replica.clone()); + db.fence().force_open_for_tests(chrono::Utc::now()); + let cid = CommunityId::from_uuid(community); + + let head = db + .get_channel_window(cid, channel, 1, None, None) + .await + .expect("head window"); + let cursor = head.next_cursor.expect("has_more implies next_cursor"); + let (_window, mut session) = db + .get_channel_window_with_session(cid, channel, 10, Some(cursor), None) + .await + .expect("routed cursor window"); + assert!( + session.is_replica(), + "fixture must route this page to the replica" + ); + + // Kill the reader's backend out from under the held transaction. + sqlx::query( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity \ + WHERE datname = $1 AND pid <> pg_backend_pid()", + ) + .bind(&rname) + .execute(&admin) + .await + .expect("terminate replica backends"); + + let mut aux = EventQuery::for_community(cid); + aux.channel_id = Some(channel); + let rows = session + .query_events(&aux) + .await + .expect("session must degrade to the writer, not error"); + assert!( + rows.iter() + .any(|se| se.event.content == "fresh-writer-only"), + "degraded aux must be served by the writer" + ); + assert!( + !session.is_replica(), + "the session must be permanently degraded to the writer" + ); + + drop(session); + drop_scratch_db(&admin, replica, &rname).await; + drop_scratch_db(&admin, writer, &wname).await; + } + /// Snapshot continuity (Wren, review of 17ea2ff6a): the routed request /// runs inside ONE `REPEATABLE READ, READ ONLY` transaction whose first /// statement was the heartbeat observation — so a row committed on the From 9fa3c9c0bdb9b1f293f81d1f8ccb542f408b6e9e Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Wed, 29 Jul 2026 16:10:35 -0400 Subject: [PATCH 8/9] Route feed, by-ids, and COUNT reads to the replica; size and lazily connect the reader pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New routed seams (all Bounded-only; deploy-default dark until BUZZ_REPLICA_READ_MAX_AGE_MS is set): - query_events_routed / query_events_routed_bounded / count_events_routed - get_events_by_ids_routed (req + bridge search hydration) - query_feed_{mentions,needs_action,activity}_routed (bridge feed) Reader pool (D4/D5): - connect_lazy + min_connections(0): reader-down at boot cannot crash the relay; boot-time reachability is a warn-only one-shot ping that also primes the Aurora identity capability cache so the first routed read spends a single acquire budget - READER_ACQUIRE_TIMEOUT = 150ms; an acquire miss fails closed to the writer as reason "reader_acquire_timeout" — named for the mechanism, not a diagnosis, because the budget includes cold connects and sqlx's size counter includes in-flight dials (see proved_reader doc-comment) - BUZZ_DB_READ_POOL_SIZE sizes the reader independently; invalid or zero falls back to the writer sizing - read_pool_stats().max reports the reader's own ceiling Tests: routed-seam truth table, lazy-pool wiring, and PG-gated end-to-end proofs including a seven-seam two-community isolation fixture (mutation-tested) proving replica-served reads are confined to the requested community. Design doc: PLANS/REPLICA_FULL_READ_ROUTING_DESIGN.md Rev 6. Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- crates/buzz-db/src/event.rs | 27 +- crates/buzz-db/src/feed.rs | 62 +- crates/buzz-db/src/lib.rs | 1321 +++++++++++++++++++++-- crates/buzz-db/src/replica_fence.rs | 25 +- crates/buzz-relay/src/api/bridge.rs | 34 +- crates/buzz-relay/src/config.rs | 120 +- crates/buzz-relay/src/handlers/count.rs | 16 +- crates/buzz-relay/src/handlers/req.rs | 4 +- crates/buzz-relay/src/main.rs | 9 +- 9 files changed, 1474 insertions(+), 144 deletions(-) diff --git a/crates/buzz-db/src/event.rs b/crates/buzz-db/src/event.rs index 70eaa5c28a..0e54196d11 100644 --- a/crates/buzz-db/src/event.rs +++ b/crates/buzz-db/src/event.rs @@ -607,6 +607,14 @@ pub(crate) fn row_to_stored_event(row: sqlx::postgres::PgRow) -> Result Result { + let mut conn = pool.acquire().await?; + count_events_on(&mut conn, q).await +} + +/// [`count_events`] on a specific session — the replica-routing path runs +/// the count on the exact reader connection whose heartbeat observation +/// proved its predicate. +pub(crate) async fn count_events_on(conn: &mut sqlx::PgConnection, q: &EventQuery) -> Result { // Empty list means "match nothing" — return 0 immediately. if q.kinds.as_deref().is_some_and(|k| k.is_empty()) { return Ok(0); @@ -741,7 +749,7 @@ pub async fn count_events(pool: &PgPool, q: &EventQuery) -> Result { } } - let row = qb.build().fetch_one(pool).await?; + let row = qb.build().fetch_one(&mut *conn).await?; let cnt: i64 = row.try_get("cnt")?; Ok(cnt) @@ -1001,6 +1009,21 @@ pub async fn get_events_by_ids( pool: &PgPool, community_id: CommunityId, ids: &[&[u8]], +) -> Result> { + if ids.is_empty() { + return Ok(vec![]); + } + let mut conn = pool.acquire().await?; + get_events_by_ids_on(&mut conn, community_id, ids).await +} + +/// [`get_events_by_ids`] on a specific session — the replica-routing path +/// runs the query on the exact reader connection whose heartbeat +/// observation proved its predicate. +pub(crate) async fn get_events_by_ids_on( + conn: &mut sqlx::PgConnection, + community_id: CommunityId, + ids: &[&[u8]], ) -> Result> { if ids.is_empty() { return Ok(vec![]); @@ -1019,7 +1042,7 @@ pub async fn get_events_by_ids( } qb.push(")"); - let rows = qb.build().fetch_all(pool).await?; + let rows = qb.build().fetch_all(&mut *conn).await?; let mut out = Vec::with_capacity(rows.len()); for row in rows { diff --git a/crates/buzz-db/src/feed.rs b/crates/buzz-db/src/feed.rs index 511a2a6083..40e58d0d06 100644 --- a/crates/buzz-db/src/feed.rs +++ b/crates/buzz-db/src/feed.rs @@ -132,6 +132,29 @@ pub async fn query_mentions( accessible_channel_ids: &[Uuid], since: Option>, limit: i64, +) -> Result> { + let mut conn = pool.acquire().await?; + query_mentions_on( + &mut conn, + community, + pubkey_bytes, + accessible_channel_ids, + since, + limit, + ) + .await +} + +/// [`query_mentions`] on a specific session — the replica-routing path runs +/// the query on the exact reader connection whose heartbeat observation +/// proved its predicate. +pub(crate) async fn query_mentions_on( + conn: &mut sqlx::PgConnection, + community: CommunityId, + pubkey_bytes: &[u8], + accessible_channel_ids: &[Uuid], + since: Option>, + limit: i64, ) -> Result> { let mut qb = build_mentions_query( community, @@ -140,7 +163,7 @@ pub async fn query_mentions( since, limit, ); - let rows = qb.build().fetch_all(pool).await?; + let rows = qb.build().fetch_all(&mut *conn).await?; collect_stored_events(rows) } @@ -193,6 +216,27 @@ pub async fn query_needs_action( accessible_channel_ids: &[Uuid], since: Option>, limit: i64, +) -> Result> { + let mut conn = pool.acquire().await?; + query_needs_action_on( + &mut conn, + community, + pubkey_bytes, + accessible_channel_ids, + since, + limit, + ) + .await +} + +/// [`query_needs_action`] on a specific session — see [`query_mentions_on`]. +pub(crate) async fn query_needs_action_on( + conn: &mut sqlx::PgConnection, + community: CommunityId, + pubkey_bytes: &[u8], + accessible_channel_ids: &[Uuid], + since: Option>, + limit: i64, ) -> Result> { let mut qb = build_needs_action_query( community, @@ -201,7 +245,7 @@ pub async fn query_needs_action( since, limit, ); - let rows = qb.build().fetch_all(pool).await?; + let rows = qb.build().fetch_all(&mut *conn).await?; collect_stored_events(rows) } @@ -241,9 +285,21 @@ pub async fn query_activity( accessible_channel_ids: &[Uuid], since: Option>, limit: i64, +) -> Result> { + let mut conn = pool.acquire().await?; + query_activity_on(&mut conn, community, accessible_channel_ids, since, limit).await +} + +/// [`query_activity`] on a specific session — see [`query_mentions_on`]. +pub(crate) async fn query_activity_on( + conn: &mut sqlx::PgConnection, + community: CommunityId, + accessible_channel_ids: &[Uuid], + since: Option>, + limit: i64, ) -> Result> { let mut qb = build_activity_query(community, accessible_channel_ids, since, limit); - let rows = qb.build().fetch_all(pool).await?; + let rows = qb.build().fetch_all(&mut *conn).await?; collect_stored_events(rows) } diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index f11ed3939c..8d3cb644ce 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -181,6 +181,13 @@ pub struct Db { /// route here (see [`Db::read`]); locks, transactions, and anything /// consistency-critical stays on `pool`. pub(crate) read_pool: Option, + /// Maximum connections configured for the read-replica pool (from + /// [`DbConfig::read_max_connections`], defaulting to the writer's + /// sizing). Kept separately from `max_connections` so + /// [`Db::read_pool_stats`] reports the reader's own ceiling — a + /// utilisation gauge derived from the writer's max would understate + /// reader saturation by exactly the ratio of the two pool sizes. + pub(crate) read_max_connections: u32, /// Freshness fence gating cursor-page routing to the replica. /// /// Starts closed; a background probe ([`replica_fence::run_probe`]) @@ -188,12 +195,13 @@ pub struct Db { /// coverage per request on the serving reader session; when the ring is /// empty or stale, every routed read stays on the writer. pub(crate) fence: std::sync::Arc, - /// Head-fetch routing budget (Predicate A): a head page may be served - /// from a proved replica session only when the proved heartbeat entry is - /// at most this old. `None` disables head routing entirely (the rollout - /// default) — bounded-stale head semantics are a product decision, not - /// an invariant, so the gate ships off. - pub(crate) replica_head_max_age: Option, + /// Bounded-staleness routing budget `B`: a read routed under + /// [`RoutePredicate::Bounded`] may be served from a proved replica + /// session only when the proved heartbeat entry is at most this old. + /// `None` disables the bounded arm entirely (the rollout default) — + /// bounded-stale read semantics are a product decision, not an + /// invariant, so the gate ships off. + pub(crate) replica_read_max_age: Option, /// Whether the reader endpoint supports the Aurora PostgreSQL identity /// function ([`replica_fence::AURORA_IDENTITY_FN`]) — probed /// once per process on the first routed read (on a plain autocommit @@ -294,38 +302,167 @@ enum RouteDecision { Writer, } +/// The ONLY place [`route_proof::ChannelScoped`] can be constructed. A +/// crate-root tuple struct would be mintable via `ChannelScoped(())` from +/// every descendant module — tuple-struct field privacy is module-scoped — +/// so the token lives in its own module and E0423 enforces the invariant. +mod route_proof { + use uuid::Uuid; + + /// Proof that a query/page can only return rows with + /// `channel_id IS NOT NULL` — the domain of the commit-time floor guard + /// (migration 0021). `channel_ids` (retains channel-NULL rows) and + /// `global_only = false` are explicitly NOT proofs. + /// + /// Each constructor keys off *how* its path proves channel-bearing-ness: + /// a pinned query filter, a bare `Uuid` argument, or a `NOT NULL` column + /// reached through an inner join. Do not add a universal constructor + /// callers reshape their inputs to fit, and never fabricate a throwaway + /// `EventQuery` purely to mint a token — the proof must be the SQL's + /// shape, not "someone assembled a struct". + #[derive(Clone, Copy)] + pub(crate) struct ChannelScoped(()); + + impl ChannelScoped { + /// Constructor 1: the query pins a single channel + /// (`EventQuery.channel_id = Some(_)`, compiled to a + /// `channel_id = $n` predicate). This proof covers BOTH query + /// builders — the SELECT builder (`event::query_events_on`) and the + /// COUNT builder (`event::count_events`) pin identically; if the + /// two ever drift, this comment is a lie and the routed COUNT seam + /// is unsound. + /// Sound under conjunction: any additional clause (e.g. + /// `channel_ids`, which alone retains channel-NULL rows) is ANDed, + /// and `channel_id = ` never matches NULL — the pin strictly + /// narrows and cannot be widened back out to global rows. + pub(crate) fn from_pinned_channel(q: &crate::event::EventQuery) -> Option { + q.channel_id.map(|_| ChannelScoped(())) + } + + /// Constructor 2 (thread pages): the page is an inner JOIN from + /// `thread_metadata` to `events`, and `thread_metadata.channel_id` + /// is `UUID NOT NULL` — every writer that creates a row passes a + /// concrete channel (`ThreadMetadataParams.channel_id: Uuid`, + /// non-Option). Channel-bearing by construction of the join, not by + /// query predicate. + pub(crate) fn from_thread_metadata_join() -> Self { + ChannelScoped(()) + } + + /// Constructor 3 (channel windows): the channel arrives as a bare + /// `Uuid` argument and the SQL binds it unconditionally + /// (`e.channel_id = $2` in `get_channel_window_on`); every served + /// row is channel-bearing. No `EventQuery` exists on this path. + pub(crate) fn from_channel_id(_channel_id: Uuid) -> Self { + ChannelScoped(()) + } + } +} +use route_proof::ChannelScoped; + /// The predicate one routed read must satisfy (see [`Db::route_read`]). +/// +/// Discipline: no `Default`, no `Deserialize`, stays non-`pub` — any of +/// those re-opens the [`ChannelScoped`] mint. enum RoutePredicate { - /// Predicate A — head fetch, bounded staleness: the proved entry must - /// be within the configured head budget (default off). - Head, - /// Predicate B — cursor page, completeness: the proved wall must cover - /// the cursor timestamp. `None` means the caller could not derive an - /// upper bound from its cursor (forward-walking thread pages) and - /// post-verifies the served rows against the proved wall itself. - Cursor(Option>), + /// Bounded staleness: the proved entry must be within the configured + /// read budget `B` (default off). Bounds TIME — the page misses at most + /// the freshest `B` of writes. Sound for ANY query shape, including + /// global (channel-NULL) rows: it relies only on heartbeat commit order, + /// not the floor guard. + Bounded, + /// Completeness: the proved wall must cover the page's upper bound. + /// Bounds CONTENT — every row at/below `upper` is present, meaningful + /// even when the cursor is hours old, where `B`-freshness says nothing. + /// Sound ONLY on the floor guard's domain (channel-bearing rows), hence + /// the proof token. `upper` is non-optional: the no-upper-bound + /// post-verifying case is [`RoutePredicate::CoveredPostVerified`]. + /// + /// Bounds INSERT-completeness only — "no missing rows", not "no extra + /// rows". Soft deletes are `UPDATE .. SET deleted_at` commits outside + /// the floor guard and never touch `created_at`, so a covered page can + /// briefly serve a row the writer already excludes; deletion visibility + /// is bounded by replication lag under `FENCE_STALENESS` (30s), not by + /// `upper` or `B`. Do not extend the covered arm to a surface that + /// cannot absorb extra rows (this is why the routed COUNT seam is + /// bounded-only). + Covered { + upper: DateTime, + /// Never read — the field exists so constructing this variant + /// requires minting the token through `route_proof`. + #[allow(dead_code)] + proof: ChannelScoped, + }, + /// Forward-walking thread pages: no upper bound is derivable from the + /// cursor; the caller post-verifies the served rows against the proved + /// wall (full page + tail at/below the wall, else re-run on the writer). + /// Only the thread path constructs this — a general routed caller does + /// no post-verification and must never self-certify. + CoveredPostVerified { + #[allow(dead_code)] + proof: ChannelScoped, + }, + /// Either arm admits, covered tried first (it has no budget dependence). + /// For general routed reads that are channel-pinned AND carry an + /// `until` upper bound. + BoundedOrCovered { + upper: DateTime, + /// Never read — see [`RoutePredicate::Covered::proof`]. + #[allow(dead_code)] + proof: ChannelScoped, + }, } impl RoutePredicate { - /// A channel-window request: cursor pages are bounded above by the - /// cursor timestamp (Predicate B); a head fetch is Predicate A. - fn from_channel_cursor(cursor: &Option<(DateTime, Vec)>) -> Self { + /// A channel-window request: cursor pages are covered-only — for deep + /// keyset pages only coverage answers "have all rows below the cursor + /// replayed?" — and a head fetch is bounded. The channel id is the + /// bare-`Uuid` proof that the window SQL pins a channel. + fn from_channel_cursor(channel_id: Uuid, cursor: &Option<(DateTime, Vec)>) -> Self { match cursor { - Some((ts, _)) => RoutePredicate::Cursor(Some(*ts)), - None => RoutePredicate::Head, + Some((ts, _)) => RoutePredicate::Covered { + upper: *ts, + proof: ChannelScoped::from_channel_id(channel_id), + }, + None => RoutePredicate::Bounded, + } + } + + /// General entry point for the routed query seams: derives the strongest + /// sound predicate from the query shape. Never produces a covered arm + /// without both a channel-scope proof AND a real upper bound. + /// + /// `routing_enabled` is whether `BUZZ_REPLICA_READ_MAX_AGE_MS` is set + /// (non-zero). When it is NOT, this returns `Bounded` — which the zero + /// budget then fails closed — so the new seams are genuinely dark at + /// the deploy default even for channel-pinned queries carrying `until`. + /// Without this gate, `BoundedOrCovered` would take the covered arm + /// (which has no budget dependence) and route on day one with no env + /// var set and no kill switch short of removing the replica URL + /// (Dawn's covered-at-zero-budget catch). The pre-existing cursor + /// paths (`Covered`/`CoveredPostVerified` from channel windows and + /// thread pages) intentionally still route at B=0 — status quo, + /// unchanged. + fn for_query(q: &event::EventQuery, routing_enabled: bool) -> Self { + if !routing_enabled { + return RoutePredicate::Bounded; + } + match (ChannelScoped::from_pinned_channel(q), q.until) { + (Some(proof), Some(upper)) => RoutePredicate::BoundedOrCovered { upper, proof }, + _ => RoutePredicate::Bounded, } } } -/// Map the configured head budget (`BUZZ_REPLICA_HEAD_MAX_AGE_SECS`) to the -/// runtime gate: `0` disables head routing; anything above the fence -/// staleness gate is clamped to it (an entry older than the staleness gate -/// never routes anyway, so a larger budget would only misrepresent the +/// Map the configured read budget (`BUZZ_REPLICA_READ_MAX_AGE_MS`) to the +/// runtime gate: `0` disables bounded-staleness routing; anything above the +/// fence staleness gate is clamped to it (an entry older than the staleness +/// gate never routes anyway, so a larger budget would only misrepresent the /// config). -fn head_budget_from_secs(secs: u64) -> Option { - match secs { +fn read_budget_from_ms(ms: u64) -> Option { + match ms { 0 => None, - secs => Some(Duration::from_secs(secs).min(replica_fence::FENCE_STALENESS)), + ms => Some(Duration::from_millis(ms).min(replica_fence::FENCE_STALENESS)), } } @@ -372,6 +509,9 @@ pub struct DbConfig { pub read_database_url: Option, /// Maximum number of connections in the pool. pub max_connections: u32, + /// Maximum connections in the read-replica pool (env + /// `BUZZ_DB_READ_POOL_SIZE`). `None` inherits [`Self::max_connections`]. + pub read_max_connections: Option, /// Minimum number of idle connections to maintain. pub min_connections: u32, /// Seconds to wait when acquiring a connection before timing out. @@ -380,12 +520,13 @@ pub struct DbConfig { pub max_lifetime_secs: u64, /// Seconds a connection may sit idle before being closed. pub idle_timeout_secs: u64, - /// Head-fetch replica budget `B` in seconds (Predicate A, env - /// `BUZZ_REPLICA_HEAD_MAX_AGE_SECS`). `0` disables head routing — the - /// rollout default. Values above [`replica_fence::FENCE_STALENESS`] are - /// clamped to it: an entry older than the staleness gate never routes - /// anyway, so a larger budget would only misrepresent the config. - pub replica_head_max_age_secs: u64, + /// Replica read budget `B` in milliseconds (bounded arm, env + /// `BUZZ_REPLICA_READ_MAX_AGE_MS`). `0` disables bounded-staleness + /// routing — the rollout default. Values above + /// [`replica_fence::FENCE_STALENESS`] are clamped to it: an entry older + /// than the staleness gate never routes anyway, so a larger budget + /// would only misrepresent the config. + pub replica_read_max_age_ms: u64, } impl Default for DbConfig { @@ -397,11 +538,12 @@ impl Default for DbConfig { database_url: "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string(), // sadscan:disable np.postgres.1 read_database_url: None, max_connections: 20, + read_max_connections: None, min_connections: 2, acquire_timeout_secs: 3, max_lifetime_secs: 1800, idle_timeout_secs: 600, - replica_head_max_age_secs: 0, + replica_read_max_age_ms: 0, } } } @@ -508,17 +650,21 @@ impl Db { /// proof hold for every insert path that goes through this pool. pub async fn new(config: &DbConfig) -> Result { let pool = Self::connect_pool(config, &config.database_url, true).await?; + let read_max_connections = config + .read_max_connections + .unwrap_or(config.max_connections); let read_pool = match &config.read_database_url { - Some(url) => Some(Self::connect_pool(config, url, false).await?), + Some(url) => Some(Self::connect_read_pool(config, url, read_max_connections)?), None => None, }; - let replica_head_max_age = head_budget_from_secs(config.replica_head_max_age_secs); + let replica_read_max_age = read_budget_from_ms(config.replica_read_max_age_ms); Ok(Self { pool, max_connections: config.max_connections, read_pool, + read_max_connections, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), - replica_head_max_age, + replica_read_max_age, reader_aurora_identity: std::sync::Arc::new(std::sync::OnceLock::new()), }) } @@ -551,14 +697,88 @@ impl Db { Ok(options.connect(url).await?) } + /// Reader acquire timeout — deliberately far below the writer's + /// (seconds-denominated) timeout. Failing closed to the writer must be + /// fast: a saturated reader pool that made routed reads wait the full + /// writer-style timeout would add dead latency during exactly the load + /// spike the offload exists for. A miss here surfaces as + /// `writer/reader_acquire_timeout` (see [`Db::proved_reader`] for why + /// the reason names the mechanism rather than a diagnosis). + const READER_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(150); + + /// Connect the read-replica pool **lazily** — no connection is + /// attempted at construction, so a reader that is down at boot cannot + /// crash the relay (it starts all-writer with the fence closed and + /// recovers when the replica returns). + /// + /// `min_connections` is pinned to 0 explicitly: sqlx's lazy pool still + /// spawns an eager background connect task to satisfy a nonzero + /// minimum, which would reintroduce boot-time reader dial attempts (and + /// their log noise) that "lazy" is meant to avoid. With 0, connections + /// are dialed only on first acquire; the ~10-minute reaper never tops + /// the pool back up, which is fine — routed reads re-fill it on demand. + /// + /// No floor guard: replica sessions are read-only, the trigger never + /// fires there (see [`Db::connect_pool`]). + fn connect_read_pool(config: &DbConfig, url: &str, max_connections: u32) -> Result { + Ok(PgPoolOptions::new() + .max_connections(max_connections) + .min_connections(0) + .acquire_timeout(Self::READER_ACQUIRE_TIMEOUT) + .max_lifetime(Duration::from_secs(config.max_lifetime_secs)) + .idle_timeout(Duration::from_secs(config.idle_timeout_secs)) + .connect_lazy(url)?) + } + + /// Spawn a one-shot reader reachability probe that only WARNs. + /// + /// With a lazy pool and `min_connections(0)`, nothing dials the replica + /// until the first routed read — so a misconfigured `READ_DATABASE_URL` + /// would otherwise be invisible until traffic arrives and quietly falls + /// back to the writer. This ping is the only boot-time reader-down + /// visibility; it must never gate startup or [`Db::spawn_fence_probe`]. + /// + /// On success it also primes the Aurora identity capability cache + /// ([`Db::reader_aurora_identity`]) on the connection it already holds, + /// so the first routed read doesn't spend a second acquire (up to + /// another [`Db::READER_ACQUIRE_TIMEOUT`]) inside + /// [`Db::reader_aurora_capability`]. Prime failure is fine: the inline + /// probe remains as the retry path. + pub fn spawn_read_pool_boot_ping(&self) { + let Some(read_pool) = self.read_pool.clone() else { + return; + }; + let aurora_identity = self.reader_aurora_identity.clone(); + tokio::spawn(async move { + match read_pool.acquire().await { + Ok(mut conn) => { + tracing::info!("read replica reachable at boot"); + match replica_fence::reader_supports_aurora_identity(&mut conn).await { + Ok(supported) => { + let _ = aurora_identity.set(supported); + } + Err(e) => tracing::debug!( + error = %e, + "aurora identity boot prime failed; first routed read will probe" + ), + } + } + Err(e) => tracing::warn!( + "read replica unreachable at boot; serving all-writer until it recovers: {e}" + ), + } + }); + } + /// Creates a `Db` from an existing `PgPool` (useful in tests). pub fn from_pool(pool: PgPool) -> Self { Self { max_connections: pool.options().get_max_connections(), + read_max_connections: pool.options().get_max_connections(), pool, read_pool: None, fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), - replica_head_max_age: None, + replica_read_max_age: None, reader_aurora_identity: std::sync::Arc::new(std::sync::OnceLock::new()), } } @@ -573,18 +793,19 @@ impl Db { pub fn from_pools(pool: PgPool, read_pool: PgPool) -> Self { Self { max_connections: pool.options().get_max_connections(), + read_max_connections: read_pool.options().get_max_connections(), pool, read_pool: Some(read_pool), fence: std::sync::Arc::new(replica_fence::ReplicaFence::new()), - replica_head_max_age: None, + replica_read_max_age: None, reader_aurora_identity: std::sync::Arc::new(std::sync::OnceLock::new()), } } /// Test hook: set the head-fetch routing budget (Predicate A), which /// [`Db::from_pools`] leaves disabled. - pub fn set_replica_head_max_age_for_tests(&mut self, budget: Option) { - self.replica_head_max_age = budget; + pub fn set_replica_read_max_age_for_tests(&mut self, budget: Option) { + self.replica_read_max_age = budget; } /// The freshness fence gating replica routing (see [`replica_fence`]). @@ -671,6 +892,33 @@ impl Db { .await { Ok(tx) => tx, + // The acquire miss gets its own reason code: the reader pool's + // short acquire timeout (READER_ACQUIRE_TIMEOUT) makes this the + // fast fail-closed path under load, and + // `buzz_db_route_decision{decision="writer",reason="reader_acquire_timeout"}` + // is the operator's alert signal for a struggling reader pool. + // + // The reason deliberately names the mechanism, not a diagnosis: + // `PoolTimedOut` proves only that no connection was handed out + // within the 150ms budget. That budget includes cold connect + // (TCP+TLS+auth), and sqlx's `size` counts in-flight dials, so + // this fires for slow connection establishment as well as for + // established-connection contention — and neither `size == 0` + // nor `size >= max` recovers the missing causal bit (in-flight + // dials hold a size slot, and a cold burst can push + // `active = size - idle` toward max with zero busy connections). + // Runbook: correlate with `buzz_db_read_pool_active` / `_max` + // and reader connection health/latency; high active suggests + // contention, but this metric alone does not distinguish + // contention from slow connects. Note the gauge is a coarse + // sample (BUZZ_POOL_METRICS_INTERVAL_SECS, default 10s) while + // the event it explains lasts ~150ms — a short burst may fall + // between samples entirely, so absence of elevated active is + // NOT evidence of a cold connect. + Err(sqlx::Error::PoolTimedOut) => { + tracing::warn!("reader pool acquire timed out; routing to writer"); + return Err("reader_acquire_timeout"); + } Err(e) => { tracing::warn!(error = %e, "reader transaction begin failed; routing to writer"); return Err("reader_validation_error"); @@ -759,11 +1007,18 @@ impl Db { } /// Pool utilisation stats for the read-replica pool, when configured. + /// + /// `max` is the **reader's** ceiling ([`Db::read_max_connections`]), not + /// the writer's: `buzz_db_read_pool_active / buzz_db_read_pool_max` is + /// the operator's utilisation signal for tuning `BUZZ_DB_READ_POOL_SIZE`, + /// and deriving it from the writer's max would misreport saturation by + /// exactly the ratio of the two pool sizes — in the direction that hides + /// the problem. pub fn read_pool_stats(&self) -> Option { self.read_pool.as_ref().map(|p| DbPoolStats { size: p.size(), idle: p.num_idle() as u32, - max: self.max_connections, + max: self.read_max_connections, }) } @@ -1351,15 +1606,126 @@ impl Db { } /// Queries events matching the given filter parameters. + /// + /// Always reads from the WRITER pool. If the result influences a write + /// or a permission decision, this is the method to call. Display-path + /// callers that tolerate bounded staleness should use + /// [`Db::query_events_routed`] instead — converting a caller is an + /// explicit, per-callsite decision, never a change to this method. pub async fn query_events(&self, q: &EventQuery) -> Result> { event::query_events(&self.pool, q).await } + /// [`Db::query_events`] with replica routing — the opt-in fast path for + /// display reads. + /// + /// Rule of thumb: **if the result influences a write or a permission, + /// it reads from the writer** — do not convert such a caller to this + /// method. Every new caller must be added to the caller-classification + /// table in `PLANS/REPLICA_FULL_READ_ROUTING_DESIGN.md`. + /// + /// Routing derives the strongest sound predicate from the query shape + /// ([`RoutePredicate::for_query`]): a channel-pinned query with an + /// `until` upper bound may be served covered (provably complete below + /// the fence wall); anything else is bounded-staleness only. The whole + /// seam is gated on `BUZZ_REPLICA_READ_MAX_AGE_MS` (default off): when + /// unset, even covered-eligible queries stay on the writer, so merging + /// this seam is a true no-op until the budget is configured. Every + /// failure fails closed to the writer. + pub async fn query_events_routed( + &self, + path: &'static str, + q: &EventQuery, + ) -> Result> { + let predicate = RoutePredicate::for_query(q, self.replica_read_max_age.is_some()); + match self.route_read(path, predicate).await { + RouteDecision::Replica(mut tx, _entry, reason) => { + match event::query_events_on(&mut tx, q).await { + Ok(events) => { + Self::record_route(path, "replica", reason); + Ok(events) + } + Err(e) => { + // Mid-query replica failure: fail closed to the + // writer rather than surfacing a routed error. + tracing::warn!(path, "replica read failed; re-running on writer: {e}"); + Self::record_route(path, "writer", "replica_error"); + event::query_events(&self.pool, q).await + } + } + } + RouteDecision::Writer => event::query_events(&self.pool, q).await, + } + } + + /// [`Db::query_events_routed`] restricted to the BOUNDED arm — for + /// reads whose result feeds a COUNT rather than a displayed page. + /// + /// The covered arm bounds insert-completeness only; stale deletions can + /// briefly inflate the result set (see [`RoutePredicate::Covered`]). A + /// display page absorbs that per-row; a number derived from the rows + /// does not. Same classification-table requirement as + /// [`Db::query_events_routed`]. + pub async fn query_events_routed_bounded( + &self, + path: &'static str, + q: &EventQuery, + ) -> Result> { + match self.route_read(path, RoutePredicate::Bounded).await { + RouteDecision::Replica(mut tx, _entry, reason) => { + match event::query_events_on(&mut tx, q).await { + Ok(events) => { + Self::record_route(path, "replica", reason); + Ok(events) + } + Err(e) => { + tracing::warn!(path, "replica read failed; re-running on writer: {e}"); + Self::record_route(path, "writer", "replica_error"); + event::query_events(&self.pool, q).await + } + } + } + RouteDecision::Writer => event::query_events(&self.pool, q).await, + } + } + /// Count events matching the given query (NIP-45 COUNT support). + /// + /// Always reads from the WRITER pool — see [`Db::query_events`] for the + /// writer-vs-routed rule. pub async fn count_events(&self, q: &EventQuery) -> Result { event::count_events(&self.pool, q).await } + /// [`Db::count_events`] with replica routing — same contract, rules, + /// and classification-table requirement as [`Db::query_events_routed`]. + /// + /// Counts route on the BOUNDED arm only, never covered: the covered + /// arm bounds insert-completeness but not deletion visibility (soft + /// deletes are UPDATEs outside the floor guard), and a count has no + /// downstream per-row re-filter to absorb extra rows — a silently + /// inflated number for up to `FENCE_STALENESS` is a different product + /// statement than a page briefly showing a deleted row. `Bounded` ties + /// the error to the accepted budget `B`. + pub async fn count_events_routed(&self, path: &'static str, q: &EventQuery) -> Result { + match self.route_read(path, RoutePredicate::Bounded).await { + RouteDecision::Replica(mut tx, _entry, reason) => { + match event::count_events_on(&mut tx, q).await { + Ok(count) => { + Self::record_route(path, "replica", reason); + Ok(count) + } + Err(e) => { + tracing::warn!(path, "replica count failed; re-running on writer: {e}"); + Self::record_route(path, "writer", "replica_error"); + event::count_events(&self.pool, q).await + } + } + } + RouteDecision::Writer => event::count_events(&self.pool, q).await, + } + } + /// Return whether a creator-signed huddle-start event links a parent /// channel to an ephemeral huddle channel. pub async fn huddle_started_link_exists( @@ -1479,6 +1845,37 @@ impl Db { event::get_events_by_ids(&self.pool, community_id, ids).await } + /// [`Db::get_events_by_ids`] with replica routing — same contract and + /// classification-table requirement as [`Db::query_events_routed`]. + /// + /// By-id fetches route on the BOUNDED arm only: an id list carries no + /// channel pin, so no fence floor can prove insert-completeness — the + /// covered arm is structurally unavailable. Used for FTS hit hydration, + /// where a missing row degrades to a skipped search hit downstream. + pub async fn get_events_by_ids_routed( + &self, + path: &'static str, + community_id: CommunityId, + ids: &[&[u8]], + ) -> Result> { + match self.route_read(path, RoutePredicate::Bounded).await { + RouteDecision::Replica(mut tx, _entry, reason) => { + match event::get_events_by_ids_on(&mut tx, community_id, ids).await { + Ok(events) => { + Self::record_route(path, "replica", reason); + Ok(events) + } + Err(e) => { + tracing::warn!(path, "replica read failed; re-running on writer: {e}"); + Self::record_route(path, "writer", "replica_error"); + event::get_events_by_ids(&self.pool, community_id, ids).await + } + } + } + RouteDecision::Writer => event::get_events_by_ids(&self.pool, community_id, ids).await, + } + } + /// Exclusively claim a batch of due matcher jobs from one community. pub async fn claim_due_push_match_batch( &self, @@ -2275,8 +2672,13 @@ impl Db { cursor: Option<&[u8]>, ) -> Result> { let (path, predicate): (&'static str, RoutePredicate) = match cursor { - Some(_) => ("thread_cursor", RoutePredicate::Cursor(None)), - None => ("thread_head", RoutePredicate::Head), + Some(_) => ( + "thread_cursor", + RoutePredicate::CoveredPostVerified { + proof: ChannelScoped::from_thread_metadata_join(), + }, + ), + None => ("thread_head", RoutePredicate::Bounded), }; if let RouteDecision::Replica(mut tx, entry, reason) = self.route_read(path, predicate).await @@ -2375,7 +2777,7 @@ impl Db { /// resolved against the fence's retained ring ([`replica_fence`]). /// - **Head fetch** (Predicate A — bounded staleness): served by a /// proved replica session only when the head gate is configured - /// ([`DbConfig::replica_head_max_age_secs`], default off) and the + /// ([`DbConfig::replica_read_max_age_ms`], default off) and the /// proved entry is within the budget. This trades a bounded staleness /// window (budget plus probe cadence) on the GET leg for writer /// offload. NOTE: enabling the budget also breaks read-your-own-writes @@ -2400,7 +2802,10 @@ impl Db { "channel_head" }; match self - .route_read(path, RoutePredicate::from_channel_cursor(&cursor)) + .route_read( + path, + RoutePredicate::from_channel_cursor(channel_id, &cursor), + ) .await { RouteDecision::Replica(mut tx, _entry, reason) => { @@ -2475,50 +2880,69 @@ impl Db { Self::record_route(path, "writer", "uninitialized"); return RouteDecision::Writer; }; - match &predicate { - // Predicate B precheck: the newest wall must cover the cursor - // (when the caller has an upper bound at all). - RoutePredicate::Cursor(Some(ts)) => { - if *ts > newest.fence_wall { - Self::record_route(path, "writer", "stale"); - return RouteDecision::Writer; + // Precheck helpers against the newest shared entry: if the newest + // cannot satisfy an arm, no proved (older-or-equal) entry can. + let bounded_precheck = + |budget: &Option| -> std::result::Result<(), &'static str> { + match budget { + Some(budget) if newest.committed_at.elapsed() <= *budget => Ok(()), + Some(_) => Err("stale"), + None => Err("disabled"), } + }; + let covered_precheck = |upper: &DateTime| -> std::result::Result<(), &'static str> { + if *upper <= newest.fence_wall { + Ok(()) + } else { + Err("stale") } - RoutePredicate::Cursor(None) => {} - // Predicate A precheck: head routing must be enabled, and the - // newest entry within budget (else no proved entry can be). - RoutePredicate::Head => match self.replica_head_max_age { - Some(budget) if newest.committed_at.elapsed() <= budget => {} - Some(_) => { - Self::record_route(path, "writer", "stale"); - return RouteDecision::Writer; - } - None => { - Self::record_route(path, "writer", "disabled"); - return RouteDecision::Writer; - } - }, + }; + let precheck = match &predicate { + RoutePredicate::Bounded => bounded_precheck(&self.replica_read_max_age), + RoutePredicate::Covered { upper, .. } => covered_precheck(upper), + // No upper bound: the caller post-verifies served rows. + RoutePredicate::CoveredPostVerified { .. } => Ok(()), + // Covered first (no budget dependence), else bounded. + RoutePredicate::BoundedOrCovered { upper, .. } => { + covered_precheck(upper).or_else(|_| bounded_precheck(&self.replica_read_max_age)) + } + }; + if let Err(reason) = precheck { + Self::record_route(path, "writer", reason); + return RouteDecision::Writer; } match self.proved_reader(read_pool).await { Ok((tx, entry)) => { - let (holds, reason): (bool, &'static str) = match &predicate { - RoutePredicate::Cursor(Some(ts)) => (*ts <= entry.fence_wall, "covered"), + // Re-evaluate against the entry the session actually proved + // (it may be older than the shared newest). + let bounded_holds = || { + self.replica_read_max_age + .is_some_and(|budget| entry.committed_at.elapsed() <= budget) + }; + let verdict: Option<&'static str> = match &predicate { + RoutePredicate::Bounded => bounded_holds().then_some("fresh"), + RoutePredicate::Covered { upper, .. } => { + (*upper <= entry.fence_wall).then_some("covered") + } // No upper bound: the caller post-verifies the served // rows against the proved wall. - RoutePredicate::Cursor(None) => (true, "covered"), - RoutePredicate::Head => ( - self.replica_head_max_age - .is_some_and(|budget| entry.committed_at.elapsed() <= budget), - "fresh", - ), + RoutePredicate::CoveredPostVerified { .. } => Some("covered"), + RoutePredicate::BoundedOrCovered { upper, .. } => { + if *upper <= entry.fence_wall { + Some("covered") + } else { + bounded_holds().then_some("fresh") + } + } }; - if holds { - RouteDecision::Replica(tx, entry, reason) - } else { - // The session proves an older entry than the predicate - // needs (replication lag) — fail closed. - Self::record_route(path, "writer", "stale"); - RouteDecision::Writer + match verdict { + Some(reason) => RouteDecision::Replica(tx, entry, reason), + None => { + // The session proves an older entry than the + // predicate needs (replication lag) — fail closed. + Self::record_route(path, "writer", "stale"); + RouteDecision::Writer + } } } Err(reason) => { @@ -2690,6 +3114,67 @@ impl Db { .await } + /// [`Db::query_feed_mentions`] with replica routing — same contract and + /// classification-table requirement as [`Db::query_events_routed`]. + /// + /// Feed queries route on the BOUNDED arm only: the `accessible_channel_ids` + /// parameter admits community-global rows alongside channel rows, so no + /// single channel's fence floor can prove completeness — the covered arm + /// is structurally unavailable, not merely unchosen. + pub async fn query_feed_mentions_routed( + &self, + path: &'static str, + community: CommunityId, + pubkey_bytes: &[u8], + accessible_channel_ids: &[Uuid], + since: Option>, + limit: i64, + ) -> Result> { + match self.route_read(path, RoutePredicate::Bounded).await { + RouteDecision::Replica(mut tx, _entry, reason) => { + match feed::query_mentions_on( + &mut tx, + community, + pubkey_bytes, + accessible_channel_ids, + since, + limit, + ) + .await + { + Ok(events) => { + Self::record_route(path, "replica", reason); + Ok(events) + } + Err(e) => { + tracing::warn!(path, "replica read failed; re-running on writer: {e}"); + Self::record_route(path, "writer", "replica_error"); + feed::query_mentions( + &self.pool, + community, + pubkey_bytes, + accessible_channel_ids, + since, + limit, + ) + .await + } + } + } + RouteDecision::Writer => { + feed::query_mentions( + &self.pool, + community, + pubkey_bytes, + accessible_channel_ids, + since, + limit, + ) + .await + } + } + } + /// Find events that require action from the given pubkey. pub async fn query_feed_needs_action( &self, @@ -2710,6 +3195,63 @@ impl Db { .await } + /// [`Db::query_feed_needs_action`] with replica routing — BOUNDED arm + /// only; see [`Db::query_feed_mentions_routed`] for why the covered arm + /// is structurally unavailable to feed queries. + pub async fn query_feed_needs_action_routed( + &self, + path: &'static str, + community: CommunityId, + pubkey_bytes: &[u8], + accessible_channel_ids: &[Uuid], + since: Option>, + limit: i64, + ) -> Result> { + match self.route_read(path, RoutePredicate::Bounded).await { + RouteDecision::Replica(mut tx, _entry, reason) => { + match feed::query_needs_action_on( + &mut tx, + community, + pubkey_bytes, + accessible_channel_ids, + since, + limit, + ) + .await + { + Ok(events) => { + Self::record_route(path, "replica", reason); + Ok(events) + } + Err(e) => { + tracing::warn!(path, "replica read failed; re-running on writer: {e}"); + Self::record_route(path, "writer", "replica_error"); + feed::query_needs_action( + &self.pool, + community, + pubkey_bytes, + accessible_channel_ids, + since, + limit, + ) + .await + } + } + } + RouteDecision::Writer => { + feed::query_needs_action( + &self.pool, + community, + pubkey_bytes, + accessible_channel_ids, + since, + limit, + ) + .await + } + } + } + /// Find recent activity across accessible channels. pub async fn query_feed_activity( &self, @@ -2721,6 +3263,53 @@ impl Db { feed::query_activity(&self.pool, community, accessible_channel_ids, since, limit).await } + /// [`Db::query_feed_activity`] with replica routing — BOUNDED arm only; + /// see [`Db::query_feed_mentions_routed`] for why the covered arm is + /// structurally unavailable to feed queries. + pub async fn query_feed_activity_routed( + &self, + path: &'static str, + community: CommunityId, + accessible_channel_ids: &[Uuid], + since: Option>, + limit: i64, + ) -> Result> { + match self.route_read(path, RoutePredicate::Bounded).await { + RouteDecision::Replica(mut tx, _entry, reason) => { + match feed::query_activity_on( + &mut tx, + community, + accessible_channel_ids, + since, + limit, + ) + .await + { + Ok(events) => { + Self::record_route(path, "replica", reason); + Ok(events) + } + Err(e) => { + tracing::warn!(path, "replica read failed; re-running on writer: {e}"); + Self::record_route(path, "writer", "replica_error"); + feed::query_activity( + &self.pool, + community, + accessible_channel_ids, + since, + limit, + ) + .await + } + } + } + RouteDecision::Writer => { + feed::query_activity(&self.pool, community, accessible_channel_ids, since, limit) + .await + } + } + } + /// Create a new API token record. #[allow(clippy::too_many_arguments)] pub async fn create_api_token( @@ -5888,19 +6477,162 @@ mod tests { } #[test] - fn head_budget_zero_disables_and_large_values_clamp_to_staleness() { - assert_eq!(head_budget_from_secs(0), None, "0 = head routing off"); + fn read_budget_zero_disables_and_large_values_clamp_to_staleness() { + assert_eq!(read_budget_from_ms(0), None, "0 = bounded routing off"); assert_eq!( - head_budget_from_secs(5), - Some(std::time::Duration::from_secs(5)) + read_budget_from_ms(1000), + Some(std::time::Duration::from_millis(1000)) ); assert_eq!( - head_budget_from_secs(10_000), + read_budget_from_ms(10_000_000), Some(replica_fence::FENCE_STALENESS), "budgets above the staleness gate clamp to it" ); } + /// Truth table for [`RoutePredicate::for_query`]: the strongest sound + /// predicate per query shape, and — the deploy-day default row — that + /// `routing_enabled = false` (BUZZ_REPLICA_READ_MAX_AGE_MS unset) + /// forces `Bounded` even for covered-eligible shapes, so the zero + /// budget fails the new seams closed (Dawn's covered-at-zero-budget + /// catch, design doc rev 5). + #[test] + fn for_query_predicate_truth_table() { + let community = CommunityId::from_uuid(Uuid::new_v4()); + let channel = Uuid::new_v4(); + let until = chrono::Utc::now(); + + let pinned_with_until = { + let mut q = event::EventQuery::for_community(community); + q.channel_id = Some(channel); + q.until = Some(until); + q + }; + let pinned_no_until = { + let mut q = event::EventQuery::for_community(community); + q.channel_id = Some(channel); + q + }; + let unpinned_with_until = { + let mut q = event::EventQuery::for_community(community); + q.until = Some(until); + q + }; + let global_only = { + let mut q = event::EventQuery::for_community(community); + q.global_only = true; + q.until = Some(until); + q + }; + + // Deploy-day default: budget unset ⇒ Bounded regardless of shape. + // The zero budget then fails Bounded closed, so the new seams + // record writer/disabled — merging with no env var set is a no-op. + assert!( + matches!( + RoutePredicate::for_query(&pinned_with_until, false), + RoutePredicate::Bounded + ), + "budget unset must not reach the covered arm even when eligible" + ); + + // Budget set + channel pin + until ⇒ the strongest predicate. + assert!(matches!( + RoutePredicate::for_query(&pinned_with_until, true), + RoutePredicate::BoundedOrCovered { .. } + )); + + // Missing either covered precondition ⇒ Bounded. + assert!(matches!( + RoutePredicate::for_query(&pinned_no_until, true), + RoutePredicate::Bounded + )); + assert!(matches!( + RoutePredicate::for_query(&unpinned_with_until, true), + RoutePredicate::Bounded + )); + // global_only implies `channel_id = None`, so the channel-pin + // precondition fails and no covered arm is possible — `for_query` + // never inspects `global_only` itself; the row holds because + // constructor 1 (channel pin) returns None for an unpinned query. + assert!(matches!( + RoutePredicate::for_query(&global_only, true), + RoutePredicate::Bounded + )); + } + + /// The pre-existing cursor paths are NOT budget-gated: a channel-window + /// cursor page still derives `Covered` with no `routing_enabled` input + /// at all — at B=0 today it routes covered, and that status quo is + /// intentionally unchanged by the `for_query` gate (Max's matrix row: + /// old paths route at budget-unset; only the new seams go dark). + #[test] + fn channel_cursor_predicate_is_not_budget_gated() { + let channel = Uuid::new_v4(); + let cursor = Some((chrono::Utc::now(), vec![1u8; 32])); + assert!(matches!( + RoutePredicate::from_channel_cursor(channel, &cursor), + RoutePredicate::Covered { .. } + )); + // Head fetch (no cursor) is bounded — gated by the budget. + assert!(matches!( + RoutePredicate::from_channel_cursor(channel, &None), + RoutePredicate::Bounded + )); + } + + /// D5 wiring: `read_pool_stats().max` must be the READER pool's own + /// ceiling, not the writer's — `buzz_db_read_pool_active / _max` is the + /// operator's utilisation signal and inheriting the writer's max hides + /// reader saturation by exactly the sizing ratio. Pure wiring test: + /// `connect_lazy` never touches the network, but it does spawn the + /// pool reaper task, which needs a Tokio runtime — hence + /// `#[tokio::test]` despite the test body itself never awaiting. + #[tokio::test] + async fn read_pool_stats_reports_reader_ceiling_not_writer() { + let writer = sqlx::postgres::PgPoolOptions::new() + .max_connections(20) + .connect_lazy(TEST_DB_URL) + .expect("lazy writer pool"); + let reader = sqlx::postgres::PgPoolOptions::new() + .max_connections(40) + .connect_lazy(TEST_DB_URL) + .expect("lazy reader pool"); + let db = Db::from_pools(writer, reader); + assert_eq!(db.pool_stats().max, 20); + assert_eq!( + db.read_pool_stats().expect("read pool configured").max, + 40, + "reader gauge must report the reader's own ceiling" + ); + } + + /// D4 wiring: the reader pool is built lazily with `min_connections(0)` + /// and the short reader acquire timeout — construction must succeed + /// with no replica listening (reader-down at boot must not crash the + /// relay), and `read_max_connections` must honour + /// `DbConfig::read_max_connections` over the writer sizing. + /// `#[tokio::test]` because `connect_lazy` spawns the pool reaper task, + /// which needs a Tokio runtime even though nothing is dialed. + #[tokio::test] + async fn connect_read_pool_is_lazy_and_independently_sized() { + let config = DbConfig { + max_connections: 20, + read_max_connections: Some(7), + ..DbConfig::default() + }; + // Unroutable per RFC 5737 TEST-NET-1: proves nothing is dialed at + // construction time. + let pool = Db::connect_read_pool(&config, "postgres://user:pw@192.0.2.1:5432/none", 7) + .expect("lazy construction must not dial the replica"); + assert_eq!(pool.options().get_max_connections(), 7); + assert_eq!(pool.options().get_min_connections(), 0); + assert_eq!( + pool.options().get_acquire_timeout(), + Db::READER_ACQUIRE_TIMEOUT + ); + } + /// Channel window: head fetch (no cursor) reads the WRITER; cursor pages /// read the REPLICA. Divergent fixtures prove which pool served each. #[tokio::test] @@ -6372,7 +7104,7 @@ mod tests { ); // Budget set, entry fresh (just recorded): head → replica. - db.set_replica_head_max_age_for_tests(Some(std::time::Duration::from_secs(5))); + db.set_replica_read_max_age_for_tests(Some(std::time::Duration::from_secs(5))); let head = db .get_channel_window(cid, channel, 2, None, None) .await @@ -6403,6 +7135,421 @@ mod tests { drop_scratch_db(&admin, writer, &wname).await; } + /// End-to-end deploy-default proof for the NEW routed seams: with the + /// budget unset, a covered-eligible query (channel-pinned + `until`) + /// through [`Db::query_events_routed`] is served by the WRITER — the + /// `for_query` gate keeps the covered arm dark (rev 5). With the budget + /// set and a fresh proved entry, the same query routes to the replica. + /// Divergent fixtures prove which pool served each read. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn query_events_routed_defaults_dark_and_routes_covered_when_enabled() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (writer, wname) = create_scratch_db(&admin, "qer_w").await; + let (replica, rname) = create_scratch_db(&admin, "qer_r").await; + + let author = nostr::Keys::generate(); + let community = Uuid::new_v4(); + let channel = Uuid::new_v4(); + seed_community_channel(&writer, community, channel, &author).await; + seed_community_channel(&replica, community, channel, &author).await; + + let base = 1_700_000_000u64; + let shared = signed_event_at(&author, "shared", base); + for pool in [&writer, &replica] { + insert_top_level(pool, community, channel, &shared).await; + } + let writer_only = signed_event_at(&author, "writer-only", base + 10); + insert_top_level(&writer, community, channel, &writer_only).await; + let replica_only = signed_event_at(&author, "replica-only", base + 20); + insert_top_level(&replica, community, channel, &replica_only).await; + + let mut db = Db::from_pools(writer.clone(), replica.clone()); + db.fence().force_open_for_tests(chrono::Utc::now()); + let cid = CommunityId::from_uuid(community); + + // Covered-eligible shape: channel-pinned with an `until` upper + // bound below the (now) fence wall. + let q = { + let mut q = EventQuery::for_community(cid); + q.channel_id = Some(channel); + q.until = chrono::DateTime::from_timestamp((base + 60) as i64, 0); + q + }; + let contents = |evs: &[StoredEvent]| -> std::collections::BTreeSet { + evs.iter().map(|e| e.event.content.clone()).collect() + }; + + // Deploy default: budget unset ⇒ writer, even though the shape is + // covered-eligible and the fence is open. + let rows = db + .query_events_routed("test_routed", &q) + .await + .expect("routed query, gate off"); + assert!( + contents(&rows).contains("writer-only"), + "budget unset must serve the writer" + ); + assert!( + !contents(&rows).contains("replica-only"), + "budget unset must not reach the replica via the covered arm" + ); + + // Budget set ⇒ the covered arm serves it from the replica. + db.set_replica_read_max_age_for_tests(Some(std::time::Duration::from_secs(5))); + let rows = db + .query_events_routed("test_routed", &q) + .await + .expect("routed query, gate on"); + assert!( + contents(&rows).contains("replica-only"), + "budget set + covered-eligible must route to the replica" + ); + assert!(!contents(&rows).contains("writer-only")); + + drop_scratch_db(&admin, replica, &rname).await; + drop_scratch_db(&admin, writer, &wname).await; + } + + /// COUNT is bounded-only (rev 5 deletion-visibility rule): a + /// covered-eligible shape must NOT let a count take the covered arm. + /// With the budget unset the count reads the WRITER even with an open + /// fence; with the budget set and a fresh entry it reads the replica + /// under the bounded arm. Divergent row counts prove the serving pool. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn count_events_routed_is_bounded_only() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (writer, wname) = create_scratch_db(&admin, "cnt_w").await; + let (replica, rname) = create_scratch_db(&admin, "cnt_r").await; + + let author = nostr::Keys::generate(); + let community = Uuid::new_v4(); + let channel = Uuid::new_v4(); + seed_community_channel(&writer, community, channel, &author).await; + seed_community_channel(&replica, community, channel, &author).await; + + let base = 1_700_000_000u64; + // Writer: 2 rows. Replica: 1 row. + for (i, content) in ["a", "b"].iter().enumerate() { + let ev = signed_event_at(&author, content, base + i as u64); + insert_top_level(&writer, community, channel, &ev).await; + } + let ev = signed_event_at(&author, "c", base); + insert_top_level(&replica, community, channel, &ev).await; + + let mut db = Db::from_pools(writer.clone(), replica.clone()); + db.fence().force_open_for_tests(chrono::Utc::now()); + let cid = CommunityId::from_uuid(community); + + // Covered-eligible shape on purpose: pinned + until. A count must + // ignore that eligibility. + let q = { + let mut q = EventQuery::for_community(cid); + q.channel_id = Some(channel); + q.until = chrono::DateTime::from_timestamp((base + 60) as i64, 0); + q + }; + + // Budget unset ⇒ bounded arm disabled ⇒ writer. + let n = db + .count_events_routed("test_count", &q) + .await + .expect("count, gate off"); + assert_eq!(n, 2, "budget unset must count on the writer"); + + // Budget set + fresh entry ⇒ bounded arm ⇒ replica. + db.set_replica_read_max_age_for_tests(Some(std::time::Duration::from_secs(5))); + let n = db + .count_events_routed("test_count", &q) + .await + .expect("count, gate on"); + assert_eq!(n, 1, "budget set must count on the replica (bounded)"); + + // Entry older than the budget ⇒ bounded fails ⇒ writer. Covered + // would still hold here (upper <= wall) — proving count never + // consults it. + db.fence().close(); + db.fence().force_open_for_tests_at( + chrono::Utc::now(), + std::time::Instant::now() - std::time::Duration::from_secs(10), + ); + let n = db + .count_events_routed("test_count", &q) + .await + .expect("count, entry too old"); + assert_eq!( + n, 2, + "an over-budget entry must fail the count closed to the writer, \ + even when the covered arm would admit the shape" + ); + + drop_scratch_db(&admin, replica, &rname).await; + drop_scratch_db(&admin, writer, &wname).await; + } + + /// Community separation across every routed seam, verified on + /// REPLICA-SERVED reads. + /// + /// The pre-existing feed/event scoping tests prove the shared SQL + /// builders confine rows to one community, but they exercise those + /// builders through the WRITER wrapper. `_on` variants are + /// executor-only refactors, so scoping *should* be identical — this + /// test refuses to take that on faith and re-proves it through the + /// routed executor, on a snapshot the replica actually served. + /// + /// Construction: two communities A and B exist in BOTH databases with + /// the same ids. The replica additionally holds a `replica-only` row in + /// each — divergent fixtures, so any row bearing that content proves + /// the replica (not the writer) served the read. Every assertion + /// requests A and demands B's rows never appear, including B's + /// `replica-only` row, which is the one a leaky predicate would surface. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn routed_reads_are_confined_to_the_requested_community() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (writer, wname) = create_scratch_db(&admin, "sep_w").await; + let (replica, rname) = create_scratch_db(&admin, "sep_r").await; + + let author = nostr::Keys::generate(); + let (comm_a, chan_a) = (Uuid::new_v4(), Uuid::new_v4()); + let (comm_b, chan_b) = (Uuid::new_v4(), Uuid::new_v4()); + for pool in [&writer, &replica] { + seed_community_channel(pool, comm_a, chan_a, &author).await; + seed_community_channel(pool, comm_b, chan_b, &author).await; + } + + // A p-tag mention is what makes a row eligible for the mentions and + // needs-action feeds. Kind 9 satisfies mentions + activity; + // needs-action admits only approval/reminder kinds, so each + // community also gets a kind-46010 row. + let mentioned = nostr::Keys::generate(); + let mentioned_hex = mentioned.public_key().to_hex(); + let mentioned_bytes = mentioned.public_key().to_bytes(); + let tagged_kind = |kind: u16, content: &str, secs: u64| { + nostr::EventBuilder::new(nostr::Kind::Custom(kind), content) + .tags([nostr::Tag::parse(["p", mentioned_hex.as_str()]).expect("p tag")]) + .custom_created_at(nostr::Timestamp::from(secs)) + .sign_with_keys(&author) + .expect("sign event") + }; + let tagged = |content: &str, secs: u64| tagged_kind(9, content, secs); + + let base = 1_700_000_000u64; + // Shared rows (both DBs) + replica-only rows (divergence) per community. + let a_shared = tagged("a-shared", base); + let b_shared = tagged("b-shared", base + 1); + for pool in [&writer, &replica] { + insert_top_level(pool, comm_a, chan_a, &a_shared).await; + insert_mentions( + pool, + CommunityId::from_uuid(comm_a), + &a_shared, + Some(chan_a), + ) + .await + .expect("mentions a-shared"); + insert_top_level(pool, comm_b, chan_b, &b_shared).await; + insert_mentions( + pool, + CommunityId::from_uuid(comm_b), + &b_shared, + Some(chan_b), + ) + .await + .expect("mentions b-shared"); + } + let a_replica_only = tagged("a-replica-only", base + 10); + let b_replica_only = tagged("b-replica-only", base + 11); + insert_top_level(&replica, comm_a, chan_a, &a_replica_only).await; + insert_mentions( + &replica, + CommunityId::from_uuid(comm_a), + &a_replica_only, + Some(chan_a), + ) + .await + .expect("mentions a-replica-only"); + insert_top_level(&replica, comm_b, chan_b, &b_replica_only).await; + insert_mentions( + &replica, + CommunityId::from_uuid(comm_b), + &b_replica_only, + Some(chan_b), + ) + .await + .expect("mentions b-replica-only"); + + // Needs-action fixtures: approval kind, replica-only in BOTH + // communities, so the assertion below is replica-served on A and + // must still not see B's. + let a_approval = tagged_kind(46010, "a-approval-replica-only", base + 20); + let b_approval = tagged_kind(46010, "b-approval-replica-only", base + 21); + insert_top_level(&replica, comm_a, chan_a, &a_approval).await; + insert_mentions( + &replica, + CommunityId::from_uuid(comm_a), + &a_approval, + Some(chan_a), + ) + .await + .expect("mentions a-approval"); + insert_top_level(&replica, comm_b, chan_b, &b_approval).await; + insert_mentions( + &replica, + CommunityId::from_uuid(comm_b), + &b_approval, + Some(chan_b), + ) + .await + .expect("mentions b-approval"); + + let mut db = Db::from_pools(writer.clone(), replica.clone()); + db.fence().force_open_for_tests(chrono::Utc::now()); + db.set_replica_read_max_age_for_tests(Some(std::time::Duration::from_secs(5))); + let cid_a = CommunityId::from_uuid(comm_a); + + let contents = |evs: &[StoredEvent]| -> std::collections::BTreeSet { + evs.iter().map(|e| e.event.content.clone()).collect() + }; + // Every routed seam must (a) have been served by the replica — + // proven by a divergent row absent from the writer — and (b) contain + // no row belonging to community B. All B fixtures are named `b-*`, + // so the leak check is a single prefix scan. + let assert_a_only = |rows: &[StoredEvent], marker: &str, seam: &str| { + let got = contents(rows); + assert!( + got.contains(marker), + "{seam}: must be replica-served (divergent row `{marker}` absent from writer); got {got:?}" + ); + assert!( + !got.iter().any(|c| c.starts_with("b-")), + "{seam}: community B rows leaked into a community A read; got {got:?}" + ); + }; + + // 1. Generic query — covered arm (channel-pinned + `until`). + let mut q = EventQuery::for_community(cid_a); + q.channel_id = Some(chan_a); + q.until = chrono::DateTime::from_timestamp((base + 60) as i64, 0); + let rows = db + .query_events_routed("sep_query", &q) + .await + .expect("routed query"); + assert_a_only(&rows, "a-replica-only", "query_events_routed"); + + // 2. Generic query — bounded arm (no channel pin at all, so a + // missing community predicate could not be masked by the pin). + let unpinned = EventQuery::for_community(cid_a); + let rows = db + .query_events_routed_bounded("sep_query_bounded", &unpinned) + .await + .expect("routed bounded query"); + assert_a_only(&rows, "a-replica-only", "query_events_routed_bounded"); + + // 3. COUNT — bounded-only. Community A holds 3 rows on the replica + // (shared + replica-only + approval) but only 1 on the writer, + // and 3 more exist in community B. Exactly 3 proves the read was + // both replica-served and community-confined. + let count = db + .count_events_routed("sep_count", &unpinned) + .await + .expect("routed count"); + assert_eq!( + count, 3, + "count must see A's three replica rows only — not B's, not the writer's one" + ); + + // 4. By-ID hydration — ids carry no channel pin, and B's ids are + // requested alongside A's. Only A's may hydrate. + let ids: Vec<&[u8]> = vec![ + a_shared.id.as_bytes(), + a_replica_only.id.as_bytes(), + b_shared.id.as_bytes(), + b_replica_only.id.as_bytes(), + ]; + let rows = db + .get_events_by_ids_routed("sep_by_ids", cid_a, &ids) + .await + .expect("routed by-ids"); + assert_a_only(&rows, "a-replica-only", "get_events_by_ids_routed"); + + // 5-7. All three feed builders, each given BOTH channels as + // accessible — so only the community predicate can exclude B. + let both = [chan_a, chan_b]; + let rows = db + .query_feed_mentions_routed("sep_feed", cid_a, &mentioned_bytes, &both, None, 50) + .await + .expect("routed mentions"); + assert_a_only(&rows, "a-replica-only", "query_feed_mentions_routed"); + + let rows = db + .query_feed_needs_action_routed("sep_feed", cid_a, &mentioned_bytes, &both, None, 50) + .await + .expect("routed needs action"); + assert_a_only( + &rows, + "a-approval-replica-only", + "query_feed_needs_action_routed", + ); + + let rows = db + .query_feed_activity_routed("sep_feed", cid_a, &both, None, 50) + .await + .expect("routed activity"); + assert_a_only(&rows, "a-replica-only", "query_feed_activity_routed"); + + drop_scratch_db(&admin, replica, &rname).await; + drop_scratch_db(&admin, writer, &wname).await; + } + + /// D4: a LAZY reader pool (connect_lazy, min_connections=0, never yet + /// used) must still let [`Db::spawn_fence_probe`] verify the writer's + /// floor guard and spawn — reader-down or reader-idle at boot must not + /// disable fence probing. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn lazy_reader_pool_still_spawns_fence_probe() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (seed, wname) = create_scratch_db(&admin, "lazy_w").await; + seed.close().await; + + let writer_url = { + let base = admin_url().await; + let idx = base.rfind('/').expect("db url has a path segment"); + format!("{}/{}", &base[..idx], wname) + }; + // `Db::new` (not `from_pools`) so the WRITER pool arms the + // `buzz.created_at_floor` GUC — `spawn_fence_probe` verifies the + // floor guard on a writer connection, and `create_scratch_db`'s + // plain `PgPool::connect` never arms it. The reader is still the + // lazy `connect_read_pool` pool this test is about. + let db = Db::new(&DbConfig { + database_url: writer_url.clone(), + read_database_url: Some(writer_url), + max_connections: 2, + ..DbConfig::default() + }) + .await + .expect("connect armed Db with lazy reader"); + + let spawned = db + .spawn_fence_probe() + .await + .expect("floor-guard verification must pass on the migrated writer"); + assert!(spawned, "a configured (lazy) reader must spawn the probe"); + + drop_scratch_db(&admin, db.pool.clone(), &wname).await; + } + /// Thread replies: head fetch reads the writer; a FULL cursor page is /// served by the replica; an UNDER-limit cursor page (candidate terminal /// page) is re-run on the writer so a lagged replica can never truncate diff --git a/crates/buzz-db/src/replica_fence.rs b/crates/buzz-db/src/replica_fence.rs index 08f8498675..c840db393e 100644 --- a/crates/buzz-db/src/replica_fence.rs +++ b/crates/buzz-db/src/replica_fence.rs @@ -81,11 +81,12 @@ pub const FENCE_CLOCK_MARGIN_SECS: i64 = 5; /// How often the probe samples the writer and commits a heartbeat token. /// -/// Rev 2 sets ~1s: the head-routing predicate (`covered_age <= B`, B -/// defaulting to 5s) needs the cadence well under the budget, or eligibility -/// flaps between beats. Cost is one single-row UPDATE tuple of WAL per beat -/// per pod. -pub const PROBE_INTERVAL: Duration = Duration::from_secs(1); +/// 500ms keeps the cadence at least 2x under the smallest sensible bounded +/// budget (`BUZZ_REPLICA_READ_MAX_AGE_MS`, deploy plan 1000ms) so +/// eligibility doesn't flap between beats. Cost is one single-row UPDATE +/// tuple of WAL per beat per pod — ~20 beats/s fleet-wide, <0.1% of the +/// writer. +pub const PROBE_INTERVAL: Duration = Duration::from_millis(500); /// A fence whose newest entry is older than this is stale: the probe has /// stopped committing tokens and routing eligibility closes until a new @@ -98,13 +99,23 @@ pub const PROBE_INTERVAL: Duration = Duration::from_secs(1); pub const FENCE_STALENESS: Duration = Duration::from_secs(30); /// How many `(token, fence_wall)` entries the fence retains. At one probe -/// per [`PROBE_INTERVAL`] this is ~2 minutes of history — a reader session -/// lagging further than that behind the newest token fails closed +/// per [`PROBE_INTERVAL`] (500ms) this is ~60 seconds of history — a reader +/// session lagging further than that behind the newest token fails closed /// (routes to the writer) rather than proving from thin air. Aurora reader /// lag is typically tens of milliseconds; a reader minutes behind is a /// fault, not a routing candidate. const RING_CAPACITY: usize = 120; +// The retained window must outlast the staleness gate: if the ring held +// less than FENCE_STALENESS of history, a non-stale newest entry could +// coexist with proved-but-evicted older entries, failing sessions closed +// for capacity rather than lag. Compile-checked so a future cadence or +// capacity tweak can't silently shrink the window below the gate. +const _: () = assert!( + RING_CAPACITY as u64 * PROBE_INTERVAL.as_millis() as u64 > FENCE_STALENESS.as_millis() as u64, + "fence ring must retain more history than the staleness gate" +); + /// One retained heartbeat observation: proof material for reader sessions. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TokenEntry { diff --git a/crates/buzz-relay/src/api/bridge.rs b/crates/buzz-relay/src/api/bridge.rs index 470332d786..10461d8d46 100644 --- a/crates/buzz-relay/src/api/bridge.rs +++ b/crates/buzz-relay/src/api/bridge.rs @@ -1087,7 +1087,8 @@ async fn query_events_authed( let type_events = match canonical { "mentions" => state .db - .query_feed_mentions( + .query_feed_mentions_routed( + "bridge_feed", tenant.community(), &pubkey_bytes, &accessible_channels, @@ -1098,7 +1099,8 @@ async fn query_events_authed( .map_err(|e| internal_error(&format!("feed mentions error: {e}")))?, "needs_action" => state .db - .query_feed_needs_action( + .query_feed_needs_action_routed( + "bridge_feed", tenant.community(), &pubkey_bytes, &accessible_channels, @@ -1109,7 +1111,13 @@ async fn query_events_authed( .map_err(|e| internal_error(&format!("feed needs_action error: {e}")))?, "activity" => state .db - .query_feed_activity(tenant.community(), &accessible_channels, since, remaining) + .query_feed_activity_routed( + "bridge_feed", + tenant.community(), + &accessible_channels, + since, + remaining, + ) .await .map_err(|e| internal_error(&format!("feed activity error: {e}")))?, _ => continue, @@ -1275,7 +1283,7 @@ async fn query_events_authed( let db = state.db.clone(); let mut catchall_results = stream::iter(catchall_queries.into_iter().map(|(idx, query)| { let db = db.clone(); - async move { (idx, db.query_events(&query).await) } + async move { (idx, db.query_events_routed("bridge_query", &query).await) } })) .buffered(crate::handlers::req::FILTER_QUERY_CONCURRENCY); @@ -1480,7 +1488,7 @@ async fn count_events_authed( && !needs_result_gated_filtering && !needs_persona_filtering { - match state.db.count_events(&query).await { + match state.db.count_events_routed("bridge_count", &query).await { Ok(n) => total += n as u64, Err(e) => { return Err(internal_error(&format!("count error: {e}"))); @@ -1490,7 +1498,11 @@ async fn count_events_authed( // Fallback: query + post-filter for non-pushable constraints. let mut q = query; crate::handlers::req::apply_count_fallback_limit(&mut q); - match state.db.query_events(&q).await { + match state + .db + .query_events_routed_bounded("bridge_count_fallback", &q) + .await + { Ok(stored_events) => { if crate::handlers::req::count_fallback_exceeded(stored_events.len()) { metrics::counter!("buzz_count_fallback_rejections_total").increment(1); @@ -1547,7 +1559,7 @@ async fn count_events_authed( && !needs_persona_filtering { query.limit = None; - match state.db.count_events(&query).await { + match state.db.count_events_routed("bridge_count", &query).await { Ok(n) => total += n as u64, Err(e) => { return Err(internal_error(&format!("count error: {e}"))); @@ -1556,7 +1568,11 @@ async fn count_events_authed( } else { // Fallback: query a bounded candidate set + post-filter. crate::handlers::req::apply_count_fallback_limit(&mut query); - match state.db.query_events(&query).await { + match state + .db + .query_events_routed_bounded("bridge_count_fallback", &query) + .await + { Ok(stored_events) => { if crate::handlers::req::count_fallback_exceeded(stored_events.len()) { metrics::counter!("buzz_count_fallback_rejections_total").increment(1); @@ -1729,7 +1745,7 @@ async fn handle_bridge_search( let id_refs: Vec<&[u8]> = hit_ids.iter().map(|b| b.as_slice()).collect(); let stored_events = state .db - .get_events_by_ids(tenant.community(), &id_refs) + .get_events_by_ids_routed("bridge_search_hydrate", tenant.community(), &id_refs) .await .map_err(|e| internal_error(&format!("search fetch error: {e}")))?; diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index 8d72d48fdc..f36befebfa 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -56,10 +56,10 @@ pub struct Config { /// Optional read-replica connection URL (e.g. an Aurora `cluster-ro-` /// endpoint). Unset means all reads stay on the writer. pub read_database_url: Option, - /// Head-fetch replica budget `B` in seconds (`BUZZ_REPLICA_HEAD_MAX_AGE_SECS`). - /// `0` (the default) disables replica routing for head fetches; see - /// [`buzz_db::DbConfig::replica_head_max_age_secs`]. - pub replica_head_max_age_secs: u64, + /// Replica read budget `B` in milliseconds (`BUZZ_REPLICA_READ_MAX_AGE_MS`). + /// `0` (the default) disables bounded-staleness replica routing; see + /// [`buzz_db::DbConfig::replica_read_max_age_ms`]. + pub replica_read_max_age_ms: u64, /// Redis connection URL used by the pub/sub manager. pub redis_url: String, /// Maximum connections in the shared Redis pool. Defaults to 16. @@ -76,6 +76,11 @@ pub struct Config { /// the per-pod pool and requests fail on acquire timeout while the /// database sits idle. pub db_pool_size: u32, + /// Maximum connections in the Postgres read-replica pool + /// (`BUZZ_DB_READ_POOL_SIZE`). Defaults to `db_pool_size`. Sized + /// independently so reader capacity can be tuned against the replica's + /// headroom without touching the writer pool. + pub db_read_pool_size: Option, /// Public WebSocket URL of this relay, advertised in NIP-11. pub relay_url: String, /// Public WebSocket URL of the dedicated device-pairing relay, when configured. @@ -427,12 +432,22 @@ impl Config { .map(|v| v.trim().to_string()) .filter(|v| !v.is_empty()); - // Head-fetch replica budget: 0 = off (the rollout default), so this - // is a non-negative parse, unlike `positive_u64_from_env`. - let replica_head_max_age_secs = match std::env::var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS") { + // The old seconds-denominated name is a hard startup error, not an + // alias: silently honouring it would mean 1000x the intended budget. + if std::env::var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS").is_ok() { + return Err(ConfigError::InvalidValue( + "BUZZ_REPLICA_HEAD_MAX_AGE_SECS was renamed to BUZZ_REPLICA_READ_MAX_AGE_MS \ + (note: milliseconds, not seconds); refusing to start" + .to_string(), + )); + } + + // Replica read budget: 0 = off (the rollout default), so this is a + // non-negative parse, unlike `positive_u64_from_env`. + let replica_read_max_age_ms = match std::env::var("BUZZ_REPLICA_READ_MAX_AGE_MS") { Ok(raw) => raw.trim().parse::().map_err(|_| { ConfigError::InvalidValue( - "BUZZ_REPLICA_HEAD_MAX_AGE_SECS must be a non-negative integer".to_string(), + "BUZZ_REPLICA_READ_MAX_AGE_MS must be a non-negative integer".to_string(), ) })?, Err(_) => 0, @@ -453,6 +468,11 @@ impl Config { .filter(|&v| v > 0) .unwrap_or(50); + let db_read_pool_size = std::env::var("BUZZ_DB_READ_POOL_SIZE") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v > 0); + let relay_url = std::env::var("RELAY_URL").unwrap_or_else(|_| "ws://localhost:3000".to_string()); @@ -902,10 +922,11 @@ impl Config { bind_addr, database_url, read_database_url, - replica_head_max_age_secs, + replica_read_max_age_ms, redis_url, redis_pool_size, db_pool_size, + db_read_pool_size, relay_url, pairing_relay_url, max_connections, @@ -1066,6 +1087,35 @@ mod tests { assert_eq!(junk, 50, "unparsable value must fall back to the default"); } + #[test] + fn db_read_pool_size_env_override_and_invalid_fallback() { + let _guard = ENV_MUTEX.lock().unwrap(); + let previous = std::env::var_os("BUZZ_DB_READ_POOL_SIZE"); + + std::env::remove_var("BUZZ_DB_READ_POOL_SIZE"); + let unset = Config::from_env().expect("config").db_read_pool_size; + + std::env::set_var("BUZZ_DB_READ_POOL_SIZE", "40"); + let overridden = Config::from_env().expect("config").db_read_pool_size; + + std::env::set_var("BUZZ_DB_READ_POOL_SIZE", "0"); + let zero = Config::from_env().expect("config").db_read_pool_size; + + std::env::set_var("BUZZ_DB_READ_POOL_SIZE", "not-a-number"); + let junk = Config::from_env().expect("config").db_read_pool_size; + + if let Some(value) = previous { + std::env::set_var("BUZZ_DB_READ_POOL_SIZE", value); + } else { + std::env::remove_var("BUZZ_DB_READ_POOL_SIZE"); + } + + assert_eq!(unset, None, "unset must inherit the writer pool sizing"); + assert_eq!(overridden, Some(40)); + assert_eq!(zero, None, "zero must fall back to inheriting"); + assert_eq!(junk, None, "unparsable value must fall back to inheriting"); + } + #[test] fn read_database_url_unset_or_blank_is_none() { let _guard = ENV_MUTEX.lock().unwrap(); @@ -1095,41 +1145,55 @@ mod tests { } #[test] - fn replica_head_max_age_defaults_off_and_rejects_junk() { + fn replica_read_max_age_defaults_off_and_rejects_junk() { let _guard = ENV_MUTEX.lock().unwrap(); - let previous = std::env::var_os("BUZZ_REPLICA_HEAD_MAX_AGE_SECS"); - + let previous = std::env::var_os("BUZZ_REPLICA_READ_MAX_AGE_MS"); + let previous_old = std::env::var_os("BUZZ_REPLICA_HEAD_MAX_AGE_SECS"); std::env::remove_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS"); - let unset = Config::from_env() - .expect("config") - .replica_head_max_age_secs; - std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", "5"); - let set = Config::from_env() - .expect("config") - .replica_head_max_age_secs; + std::env::remove_var("BUZZ_REPLICA_READ_MAX_AGE_MS"); + let unset = Config::from_env().expect("config").replica_read_max_age_ms; - std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", "0"); - let zero = Config::from_env() - .expect("config") - .replica_head_max_age_secs; + std::env::set_var("BUZZ_REPLICA_READ_MAX_AGE_MS", "1000"); + let set = Config::from_env().expect("config").replica_read_max_age_ms; - std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", "soon"); + std::env::set_var("BUZZ_REPLICA_READ_MAX_AGE_MS", "0"); + let zero = Config::from_env().expect("config").replica_read_max_age_ms; + + std::env::set_var("BUZZ_REPLICA_READ_MAX_AGE_MS", "soon"); let junk = Config::from_env(); + // The retired seconds-denominated name must be a hard startup + // error even alongside a valid new-name value: silently ignoring + // it (or honouring it) would mean 1000x the intended budget. + std::env::set_var("BUZZ_REPLICA_READ_MAX_AGE_MS", "1000"); + std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", "5"); + let old_name = Config::from_env(); + + std::env::remove_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS"); if let Some(value) = previous { - std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", value); + std::env::set_var("BUZZ_REPLICA_READ_MAX_AGE_MS", value); } else { - std::env::remove_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS"); + std::env::remove_var("BUZZ_REPLICA_READ_MAX_AGE_MS"); + } + if let Some(value) = previous_old { + std::env::set_var("BUZZ_REPLICA_HEAD_MAX_AGE_SECS", value); } - assert_eq!(unset, 0, "head routing must default off"); - assert_eq!(set, 5); + assert_eq!(unset, 0, "replica read routing must default off"); + assert_eq!(set, 1000); assert_eq!(zero, 0, "explicit 0 is off"); assert!( junk.is_err(), "an unparsable budget must fail loudly, not silently disable" ); + match old_name { + Err(ConfigError::InvalidValue(message)) => assert!( + message.contains("BUZZ_REPLICA_READ_MAX_AGE_MS"), + "the error must name the replacement env var, got: {message}" + ), + other => panic!("old env name must hard-fail startup, got {other:?}"), + } } #[test] diff --git a/crates/buzz-relay/src/handlers/count.rs b/crates/buzz-relay/src/handlers/count.rs index 614e54d7a0..dfb44e152f 100644 --- a/crates/buzz-relay/src/handlers/count.rs +++ b/crates/buzz-relay/src/handlers/count.rs @@ -173,7 +173,7 @@ pub async fn handle_count( && !needs_result_gated_filtering && !needs_persona_filtering { - match state.db.count_events(&query).await { + match state.db.count_events_routed("count_req", &query).await { Ok(n) => total += n as u64, Err(e) => { conn.send(RelayMessage::closed(&sub_id, &format!("error: {e}"))); @@ -184,7 +184,11 @@ pub async fn handle_count( // Fallback: query + post-filter for non-pushable constraints. let mut q = query; super::req::apply_count_fallback_limit(&mut q); - match state.db.query_events(&q).await { + match state + .db + .query_events_routed_bounded("count_req_fallback", &q) + .await + { Ok(stored_events) => { if super::req::count_fallback_exceeded(stored_events.len()) { metrics::counter!("buzz_count_fallback_rejections_total").increment(1); @@ -243,7 +247,7 @@ pub async fn handle_count( && !needs_persona_filtering { query.limit = None; // COUNT doesn't need a row limit - match state.db.count_events(&query).await { + match state.db.count_events_routed("count_req", &query).await { Ok(n) => total += n as u64, Err(e) => { conn.send(RelayMessage::closed(&sub_id, &format!("error: {e}"))); @@ -253,7 +257,11 @@ pub async fn handle_count( } else { // Fallback: query a bounded candidate set + post-filter. super::req::apply_count_fallback_limit(&mut query); - match state.db.query_events(&query).await { + match state + .db + .query_events_routed_bounded("count_req_fallback", &query) + .await + { Ok(stored_events) => { if super::req::count_fallback_exceeded(stored_events.len()) { metrics::counter!("buzz_count_fallback_rejections_total").increment(1); diff --git a/crates/buzz-relay/src/handlers/req.rs b/crates/buzz-relay/src/handlers/req.rs index d3ddd3e5d3..51400452d7 100644 --- a/crates/buzz-relay/src/handlers/req.rs +++ b/crates/buzz-relay/src/handlers/req.rs @@ -310,7 +310,7 @@ pub async fn handle_req( |(idx, per_filter_channel, params)| { let db = db.clone(); async move { - let filter_events = db.query_events(¶ms).await; + let filter_events = db.query_events_routed("req_historical", ¶ms).await; (idx, per_filter_channel, filter_events) } }, @@ -629,7 +629,7 @@ async fn handle_search_req( let id_refs: Vec<&[u8]> = hit_ids.iter().map(|b| b.as_slice()).collect(); let events = match state .db - .get_events_by_ids(tenant.community(), &id_refs) + .get_events_by_ids_routed("req_search_hydrate", tenant.community(), &id_refs) .await { Ok(evs) => evs, diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 28590c3c6d..aa800d697e 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -158,8 +158,9 @@ async fn main() -> anyhow::Result<()> { let db_config = DbConfig { database_url: config.database_url.clone(), read_database_url: config.read_database_url.clone(), - replica_head_max_age_secs: config.replica_head_max_age_secs, + replica_read_max_age_ms: config.replica_read_max_age_ms, max_connections: config.db_pool_size, + read_max_connections: config.db_read_pool_size, ..DbConfig::default() }; let db = Db::new(&db_config).await.map_err(|e| { @@ -167,7 +168,11 @@ async fn main() -> anyhow::Result<()> { anyhow::anyhow!("DB connection failed: {e}") })?; if db.has_read_pool() { - info!("Postgres connected (writer + read replica)"); + info!("Postgres connected (writer + lazy read replica pool)"); + // Reader-down at boot must not crash or block the relay; this warn-only + // ping is the sole boot-time visibility that the replica is unreachable + // (the lazy pool with min_connections=0 dials nothing until first use). + db.spawn_read_pool_boot_ping(); } else { info!("Postgres connected"); } From dd26caa9fbd9052da1efdd74d753ab06c9eb0eaa Mon Sep 17 00:00:00 2001 From: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Date: Wed, 29 Jul 2026 16:55:49 -0400 Subject: [PATCH 9/9] Spend one reader acquire budget per routed read, even with a cold Aurora cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit proved_reader previously probed Aurora capability through its own pool.acquire() before begin_with acquired a second connection. With the capability cache cold — exactly the post-boot-ping-failure state the ~150ms fallback bound was specified for — a saturated reader cost two stacked READER_ACQUIRE_TIMEOUT budgets (~300ms measured 302-330ms). Now the routed path acquires once: the capability probe runs on the held connection (reader_aurora_capability_on) and the read-only REPEATABLE READ transaction begins on that same connection via Transaction::begin, which accepts a PoolConnection at 'static through sqlx's MaybePoolConnection. The pool-taking probe is deleted as dead code (the boot ping inlines its own probe). Reason codes are unchanged (PoolTimedOut -> reader_acquire_timeout, other -> reader_validation_error) and capability is still never negatively cached. Adds a PG-gated regression fixture that reproduces the cold-cache saturated-reader state (size-1 reader, sole connection held, capability asserted unprimed), routes through count_events_routed, and asserts the fallback answers from the writer within one budget while emitting writer/reader_acquire_timeout — it fails at 330ms on the previous code and passes at ~150ms on this one. metrics-util becomes a buzz-db dev-dependency for the label snapshot. Fix and fixture authored by Dawn; blocker found by Max. Design doc: PLANS/REPLICA_FULL_READ_ROUTING_DESIGN.md Rev 6. Co-authored-by: Tyler Longwell Signed-off-by: Tyler Longwell --- Cargo.lock | 1 + crates/buzz-db/Cargo.toml | 1 + crates/buzz-db/src/lib.rs | 191 +++++++++++++++++++++++++++++++++++--- 3 files changed, 182 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2fc3158a02..bd9c29fd2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -950,6 +950,7 @@ dependencies = [ "chrono", "hex", "metrics", + "metrics-util", "nostr", "rand 0.10.1", "serde", diff --git a/crates/buzz-db/Cargo.toml b/crates/buzz-db/Cargo.toml index dc2304368e..6f76a11bc1 100644 --- a/crates/buzz-db/Cargo.toml +++ b/crates/buzz-db/Cargo.toml @@ -25,3 +25,4 @@ metrics = { workspace = true } [dev-dependencies] tokio = { workspace = true } +metrics-util = { workspace = true } diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 8d3cb644ce..0c6ea36dac 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -742,8 +742,9 @@ impl Db { /// ([`Db::reader_aurora_identity`]) on the connection it already holds, /// so the first routed read doesn't spend a second acquire (up to /// another [`Db::READER_ACQUIRE_TIMEOUT`]) inside - /// [`Db::reader_aurora_capability`]. Prime failure is fine: the inline - /// probe remains as the retry path. + /// [`Db::reader_aurora_capability_on`]. Prime failure is fine: the routed + /// path re-probes on the connection it already holds, so a failed prime + /// costs a round trip rather than a second acquire budget. pub fn spawn_read_pool_boot_ping(&self) { let Some(read_pool) = self.read_pool.clone() else { return; @@ -886,10 +887,32 @@ impl Db { ), &'static str, > { - let aurora = self.reader_aurora_capability(read_pool).await; - let mut tx = match read_pool - .begin_with("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY") - .await + // One checkout per routed read. The Aurora capability probe and the + // read-only transaction share a single `acquire()` so the request path + // spends exactly one READER_ACQUIRE_TIMEOUT budget. Probing through + // `read_pool` separately would spend a second budget whenever the + // capability is uncached — i.e. after a failed boot ping, which is + // precisely the reader-unavailable case the bound must hold for. + let conn = match read_pool.acquire().await { + Ok(conn) => conn, + Err(sqlx::Error::PoolTimedOut) => { + tracing::warn!("reader pool acquire timed out; routing to writer"); + return Err("reader_acquire_timeout"); + } + Err(e) => { + tracing::warn!(error = %e, "reader connection acquire failed; routing to writer"); + return Err("reader_validation_error"); + } + }; + let mut conn = conn; + let aurora = self.reader_aurora_capability_on(&mut conn).await; + let mut tx = match sqlx::Transaction::begin( + conn, + Some(sqlx::SqlStr::from_static( + "BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY", + )), + ) + .await { Ok(tx) => tx, // The acquire miss gets its own reason code: the reader pool's @@ -955,14 +978,16 @@ impl Db { /// it. Probe failure (acquire or transient) degrades to the plain /// identity tuple for THIS request without caching, so a later request /// retries; identity is evidence, never a routing gate. - async fn reader_aurora_capability(&self, read_pool: &PgPool) -> bool { + /// Aurora capability on a connection the caller already holds, so the + /// routed path never spends a second acquire budget. + async fn reader_aurora_capability_on( + &self, + conn: &mut sqlx::pool::PoolConnection, + ) -> bool { if let Some(cached) = self.reader_aurora_identity.get() { return *cached; } - let Ok(mut conn) = read_pool.acquire().await else { - return false; - }; - match replica_fence::reader_supports_aurora_identity(&mut conn).await { + match replica_fence::reader_supports_aurora_identity(conn).await { Ok(supported) => *self.reader_aurora_identity.get_or_init(|| supported), Err(e) => { tracing::debug!(error = %e, "aurora identity probe failed; will retry"); @@ -7308,6 +7333,150 @@ mod tests { /// the replica (not the writer) served the read. Every assertion /// requests A and demands B's rows never appear, including B's /// `replica-only` row, which is the one a leaky predicate would surface. + /// The routed fallback must cost ONE reader acquire budget, even when the + /// Aurora capability cache is cold. + /// + /// Regression test for a stacked-budget bug found at `9fa3c9c0b`: the + /// capability probe used to `acquire()` from the pool itself and return + /// `false` *uncached* on `PoolTimedOut`, so the routed read then spent a + /// SECOND `READER_ACQUIRE_TIMEOUT` inside `begin`. Measured 302ms against + /// a ~150ms documented bound. Boot priming + /// ([`Db::spawn_read_pool_boot_ping`]) hid it only when the boot ping + /// SUCCEEDED — and a reader that is unavailable at boot is exactly the + /// case the bound is specified for, so the two failures are correlated. + /// + /// The fixture reproduces that state deliberately: a size-1 reader whose + /// sole connection is established and then HELD (so every further acquire + /// must time out), with `reader_aurora_identity` asserted cold. It routes + /// through `count_events_routed` rather than calling `proved_reader` + /// directly, because `buzz_db_route_decision` is emitted by `route_read` + /// — a direct call would prove the timing but never emit the label. + /// + /// Timing uses an upper bound of 2x the budget minus a margin: it must + /// fail for two stacked budgets (~300ms) while tolerating scheduler + /// jitter on one (~150ms). Asserting a lower bound too would pin the + /// budget's own value, which `reader_acquire_timeout_is_the_documented_budget` + /// already covers. + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (seed, wname) = create_scratch_db(&admin, "one_budget").await; + seed.close().await; + let base = admin_url().await; + let scratch_url = { + let idx = base.rfind('/').expect("db url has a path segment"); + format!("{}/{}", &base[..idx], wname) + }; + + // `Db::new` so the writer arms the floor guard and the reader is the + // real lazy `connect_read_pool` pool (min_connections=0, 150ms + // acquire timeout). Reader is sized 1 so holding one connection + // saturates it. + let mut db = Db::new(&DbConfig { + database_url: scratch_url.clone(), + read_database_url: Some(scratch_url), + max_connections: 4, + read_max_connections: Some(1), + ..DbConfig::default() + }) + .await + .expect("connect armed Db with size-1 lazy reader"); + db.fence().force_open_for_tests(chrono::Utc::now()); + db.set_replica_read_max_age_for_tests(Some(Duration::from_secs(5))); + + let read_pool = db.read_pool.clone().expect("reader pool configured"); + // Establish and hold the reader's only connection: saturated. + let held = read_pool + .acquire() + .await + .expect("establish the reader's sole connection"); + assert_eq!( + db.read_max_connections, 1, + "reader max must report 1 for this fixture to test saturation" + ); + assert_eq!( + read_pool.size(), + 1, + "the sole reader connection is established and held" + ); + // The bug is only observable with the capability cache cold; if a + // future change primes it here, this fixture would silently stop + // discriminating. + assert!( + db.reader_aurora_identity.get().is_none(), + "Aurora capability must be UNPRIMED (post-boot-ping-failure state)" + ); + + let recorder = metrics_util::debugging::DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let query = EventQuery::for_community(CommunityId::from_uuid(Uuid::new_v4())); + + // The recorder is installed thread-locally, so it must stay installed + // across the `.await` — hence the guard form rather than + // `with_local_recorder`, whose closure cannot host an await. The + // `current_thread` flavor keeps the route decision on this thread; on + // a multi-thread runtime the emit could land on a worker where no + // local recorder is installed and the label assertions would vacuously + // see an empty snapshot. + let start = std::time::Instant::now(); + let count = { + let _guard = metrics::set_default_local_recorder(&recorder); + db.count_events_routed("one_budget_probe", &query).await + } + .expect("writer fallback still answers the read"); + let elapsed = start.elapsed(); + + assert_eq!(count, 0, "writer answered on an empty scratch database"); + assert!( + elapsed < Duration::from_millis(250), + "routed fallback must spend ONE {}ms acquire budget, not two; took {}ms", + Db::READER_ACQUIRE_TIMEOUT.as_millis(), + elapsed.as_millis() + ); + + let reasons: std::collections::HashMap<(String, String), u64> = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(key, ..)| key.key().name() == "buzz_db_route_decision") + .map(|(key, _, _, value)| { + let metrics_util::debugging::DebugValue::Counter(n) = value else { + panic!("buzz_db_route_decision must be a counter"); + }; + let labels: Vec<_> = key.key().labels().collect(); + let get = |name: &str| { + labels + .iter() + .find(|l| l.key() == name) + .map(|l| l.value().to_owned()) + .unwrap_or_default() + }; + ((get("decision"), get("reason")), n) + }) + .collect(); + + assert_eq!( + reasons.get(&("writer".to_owned(), "reader_acquire_timeout".to_owned())), + Some(&1), + "saturated reader must fall back as writer/reader_acquire_timeout; got {reasons:?}" + ); + // `reader_validation_error` would mean we misclassified a timeout as a + // broken reader, and `pool_busy` is the retired name — neither may + // appear in ANY emitted label. + assert!( + !reasons + .keys() + .any(|(_, reason)| reason == "reader_validation_error" || reason == "pool_busy"), + "no reader_validation_error or retired pool_busy label may be emitted; got {reasons:?}" + ); + + drop(held); + drop_scratch_db(&admin, db.pool.clone(), &wname).await; + } + #[tokio::test] #[ignore = "requires Postgres"] async fn routed_reads_are_confined_to_the_requested_community() {