fix(upstream): name the real transport fault and bound the connection layer

config.example.yaml

@@ -153,6 +153,37 @@ ratelimit:
   # hung upstream) is reclaimed. Redis backend only.
   # concurrency_ttl_secs: 300
 
+# Connection-layer settings for outbound calls to LLM providers. These
+# describe the network path to the upstream, so they are deployment
+# config rather than per-model or per-provider-key fields.
+#
+# The values below are the defaults; uncomment to override. Every
+# duration accepts 0 to switch that knob off.
+upstream:
+  # Max time for DNS + TCP + TLS before an attempt fails. Without it a
+  # black-holed upstream is bounded only by the model's own timeout.
+  # connect_timeout_ms: 5000
+
+  # TCP keepalive. Idle time before the first probe, the interval
+  # between probes, and how many unacknowledged probes end the
+  # connection. Keeps a NAT / LB idle timer from reaping a connection
+  # while a slow model is still producing its first token.
+  # tcp_keepalive_secs: 60
+  # tcp_keepalive_interval_secs: 30
+  # tcp_keepalive_retries: 5
+
+  # How long an idle connection may sit in the pool before being
+  # discarded. KEEP THIS BELOW the shortest idle timeout on the path to
+  # the provider (load balancer, NAT gateway, corporate proxy, service
+  # mesh). If it is longer, the pool eventually hands out a connection
+  # the far end has already closed and the request fails with a
+  # transport error. Lower it if you see intermittent transport errors
+  # against an otherwise healthy provider.
+  pool_idle_timeout_secs: 30
+
+  # Cap on idle connections kept per upstream host. Unset = unbounded.
+  # pool_max_idle_per_host: 32
+
 # Models, API keys, provider keys, guardrails, cache policies, and
 # observability exporters are NOT defined in this file. They are stored
 # in etcd and managed via the Admin API (see docs/api-admin.md). This

config.managed.yaml

@@ -99,3 +99,17 @@ managed:
 
 cache:
   backend: "memory"
+
+# Connection-layer settings for outbound calls to LLM providers. Shown
+# with their defaults; uncomment to override. `pool_idle_timeout_secs`
+# is the one to lower when a load balancer, NAT gateway, or service mesh
+# between this DP and the provider closes idle connections sooner than
+# the gateway expires them — the symptom is intermittent transport
+# errors against an otherwise healthy upstream.
+upstream:
+  # connect_timeout_ms: 5000
+  # tcp_keepalive_secs: 60
+  # tcp_keepalive_interval_secs: 30
+  # tcp_keepalive_retries: 5
+  pool_idle_timeout_secs: 30
+  # pool_max_idle_per_host: 32

crates/aisix-core/src/config.rs

@@ -61,6 +61,12 @@ pub struct Config {
     /// one global window instead of one-per-replica (api7/AISIX-Cloud#798).
     #[serde(default)]
     pub ratelimit: RateLimitConfig,
+    /// Connection-layer tuning for outbound calls to LLM providers.
+    /// Defaults bound the connect phase, keep TCP keepalive on, and expire
+    /// pooled connections well before a typical LB/NAT/proxy hop would —
+    /// see [`UpstreamConfig`].
+    #[serde(default)]
+    pub upstream: UpstreamConfig,
     /// Optional managed-mode configuration. When `managed.enabled = true`
     /// the admin API and Playground endpoints are **not** bound — the DP
     /// is a pure etcd reader driven by the aisix.cloud control plane.
@@ -755,6 +761,56 @@ pub enum RateLimitBackend {
     Redis,
 }
 
+/// Connection-layer settings for the HTTP clients that call LLM providers.
+///
+/// These are deployment properties of the network path to the upstream, not
+/// per-tenant configuration, so they live in the DP config file rather than
+/// on a Model or ProviderKey resource.
+///
+/// The defaults exist because reqwest's own are wrong for a gateway sitting
+/// behind an LB/NAT/proxy hop: no connect timeout, TCP keepalive off, and a
+/// 90s pooled-connection lifetime that outlives the idle timeout of a
+/// typical hop — so a connection reaped upstream can still be handed out
+/// here, and the request fails with an opaque transport error.
+///
+/// Every duration accepts `0` to disable that individual knob.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields, default)]
+pub struct UpstreamConfig {
+    /// Max time for DNS + TCP + TLS before an attempt fails. Without it a
+    /// black-holed upstream is bounded only by the model's overall timeout.
+    pub connect_timeout_ms: u64,
+    /// Idle seconds before the kernel sends its first TCP keepalive probe.
+    /// Keeps a long wait for a slow first token from being reaped by a NAT
+    /// or LB idle timer.
+    pub tcp_keepalive_secs: u64,
+    /// Seconds between subsequent keepalive probes.
+    pub tcp_keepalive_interval_secs: u64,
+    /// Unacknowledged probes before the kernel drops the connection.
+    pub tcp_keepalive_retries: u32,
+    /// How long an idle connection may sit in the pool before it is
+    /// discarded. **Keep this below the shortest idle timeout on the path
+    /// to the provider** (LB, NAT gateway, corporate proxy, service mesh),
+    /// or the pool will hand out connections the far end already closed.
+    pub pool_idle_timeout_secs: u64,
+    /// Cap on idle connections kept per upstream host. `null` (the
+    /// default) leaves reqwest's unbounded behaviour.
+    pub pool_max_idle_per_host: Option<usize>,
+}
+
+impl Default for UpstreamConfig {
+    fn default() -> Self {
+        Self {
+            connect_timeout_ms: 5_000,
+            tcp_keepalive_secs: 60,
+            tcp_keepalive_interval_secs: 30,
+            tcp_keepalive_retries: 5,
+            pool_idle_timeout_secs: 30,
+            pool_max_idle_per_host: None,
+        }
+    }
+}
+
 impl Config {
     /// Load + merge + validate.
     ///
@@ -1234,6 +1290,61 @@ admin:
         assert_eq!(cfg.ratelimit.concurrency_ttl_secs, 300);
     }
 
+    /// An `upstream:` block is optional; the defaults must still bound the
+    /// connect phase, keep TCP keepalive on, and expire pooled connections
+    /// sooner than reqwest's own 90s (AISIX-Cloud#1122).
+    #[test]
+    fn upstream_defaults_apply_when_the_block_is_absent() {
+        let f = write_yaml(
+            r#"
+etcd:
+  endpoints: ["http://localhost:2379"]
+proxy:
+  addr: "0.0.0.0:3000"
+admin:
+  addr: "127.0.0.1:3001"
+  admin_keys: ["k1"]
+"#,
+        );
+        let cfg = Config::load_from_path(Some(f.path())).unwrap();
+        assert_eq!(cfg.upstream.connect_timeout_ms, 5_000);
+        assert_eq!(cfg.upstream.tcp_keepalive_secs, 60);
+        assert_eq!(cfg.upstream.tcp_keepalive_interval_secs, 30);
+        assert_eq!(cfg.upstream.tcp_keepalive_retries, 5);
+        assert!(cfg.upstream.pool_idle_timeout_secs < 90);
+        assert!(cfg.upstream.pool_max_idle_per_host.is_none());
+    }
+
+    /// Operators behind a proxy with a short idle timeout need to lower
+    /// `pool_idle_timeout_secs`; every knob must be individually settable
+    /// and `0` must round-trip (it means "leave this one off").
+    #[test]
+    fn upstream_block_overrides_individual_knobs() {
+        let f = write_yaml(
+            r#"
+etcd:
+  endpoints: ["http://localhost:2379"]
+proxy:
+  addr: "0.0.0.0:3000"
+admin:
+  addr: "127.0.0.1:3001"
+  admin_keys: ["k1"]
+upstream:
+  connect_timeout_ms: 2000
+  pool_idle_timeout_secs: 10
+  tcp_keepalive_secs: 0
+  pool_max_idle_per_host: 16
+"#,
+        );
+        let cfg = Config::load_from_path(Some(f.path())).unwrap();
+        assert_eq!(cfg.upstream.connect_timeout_ms, 2_000);
+        assert_eq!(cfg.upstream.pool_idle_timeout_secs, 10);
+        assert_eq!(cfg.upstream.tcp_keepalive_secs, 0);
+        assert_eq!(cfg.upstream.pool_max_idle_per_host, Some(16));
+        // Unspecified knobs keep their defaults.
+        assert_eq!(cfg.upstream.tcp_keepalive_interval_secs, 30);
+    }
+
     #[test]
     fn ratelimit_redis_backend_requires_redis_block() {
         let f = write_yaml(

crates/aisix-gateway/src/bridge.rs

@@ -121,13 +121,32 @@ impl BridgeContext {
     }
 }
 
+/// `": {cause}"` when a transport-layer cause is known, otherwise empty —
+/// keeps the timeout message unchanged for the gateway's own deadlines.
+fn timeout_cause_suffix(cause: &str) -> String {
+    if cause.is_empty() {
+        String::new()
+    } else {
+        format!(": {cause}")
+    }
+}
+
 /// Error surfaced by any Bridge. Each variant maps to a stable
 /// client-visible HTTP status and OpenAI-style error code so the proxy
 /// layer can translate without further inspection.
 #[derive(Debug, thiserror::Error)]
 pub enum BridgeError {
-    #[error("upstream request timed out after {elapsed_ms}ms")]
-    Timeout { elapsed_ms: u64 },
+    /// An upstream call exceeded a time budget.
+    ///
+    /// `cause` names the transport-layer reason when reqwest reported one,
+    /// and is empty when one of the gateway's own deadlines elapsed. Three
+    /// unrelated conditions all satisfy `reqwest::Error::is_timeout()` — a
+    /// `connect_timeout`, hyper's request timeout, and the kernel's own
+    /// `ETIMEDOUT` on an unanswered SYN — so without the cause they render
+    /// as the same sentence and an operator cannot tell "the upstream is
+    /// slow" from "we never reached it" (AISIX-Cloud#1093).
+    #[error("upstream request timed out after {elapsed_ms}ms{}", timeout_cause_suffix(.cause))]
+    Timeout { elapsed_ms: u64, cause: String },
     /// Upstream returned a non-2xx HTTP status. `retry_after` carries
     /// the upstream's `Retry-After` header parsed to a Duration when
     /// present — used by the cooldown layer to honor provider-supplied
@@ -485,7 +504,43 @@ mod tests {
 
     #[test]
     fn timeout_maps_to_504() {
-        let e = BridgeError::Timeout { elapsed_ms: 30_000 };
+        let e = BridgeError::Timeout {
+            cause: String::new(),
+            elapsed_ms: 30_000,
+        };
+        assert_eq!(e.http_status(), 504);
+        assert_eq!(e.error_type(), "timeout");
+    }
+
+    /// A gateway-owned deadline has no transport cause, and its message
+    /// must stay byte-identical to what it was before `cause` existed —
+    /// operators and log queries key on this sentence.
+    #[test]
+    fn timeout_without_cause_renders_unchanged() {
+        let e = BridgeError::Timeout {
+            elapsed_ms: 30_000,
+            cause: String::new(),
+        };
+        assert_eq!(e.to_string(), "upstream request timed out after 30000ms");
+    }
+
+    /// With a transport cause the message names it, so an expired
+    /// `connect_timeout` and an expired request budget stop looking
+    /// identical (AISIX-Cloud#1093).
+    #[test]
+    fn timeout_with_cause_appends_it() {
+        let e = BridgeError::Timeout {
+            elapsed_ms: 5_001,
+            cause: "error sending request: client error (Connect): tcp connect error: \
+                    Connection timed out (os error 110)"
+                .to_string(),
+        };
+        assert_eq!(
+            e.to_string(),
+            "upstream request timed out after 5001ms: error sending request: \
+             client error (Connect): tcp connect error: Connection timed out (os error 110)"
+        );
+        // Status and telemetry class are unaffected by the added detail.
         assert_eq!(e.http_status(), 504);
         assert_eq!(e.error_type(), "timeout");
     }

crates/aisix-gateway/src/lib.rs

@@ -13,10 +13,12 @@
 //! - [`sse`] — a provider-agnostic SSE line decoder. Bridges that stream
 //!   over SSE feed it raw bytes and pull typed events back out.
 //! - [`credential`] — cache keys for credential-derived upstream tokens.
+//! - [`upstream_http`] — connection-layer settings every provider client
+//!   shares (connect timeout, TCP keepalive, pool expiry) plus the
+//!   cause-chain rendering for transport errors.
 //!
-//! The concrete HTTP transport lives in the provider crates — keeping
-//! this crate free of `reqwest` at the public-API level makes it testable
-//! without wiremock.
+//! Request/response translation lives in the provider crates; this crate
+//! owns only what all of them must agree on.
 
 #![forbid(unsafe_code)]
 #![deny(rust_2018_idioms)]
@@ -26,6 +28,7 @@ pub mod chat;
 pub mod credential;
 pub mod hub;
 pub mod sse;
+pub mod upstream_http;
 
 pub use bridge::{
     capture_upstream_error_http, content_type_is_json, parse_retry_after, read_body_capped,
@@ -39,3 +42,6 @@ pub use chat::{
 pub use credential::credential_fingerprint;
 pub use hub::Hub;
 pub use sse::{SseDecoder, SseEvent};
+pub use upstream_http::{
+    client_builder, error_with_causes, transport_error_message, UpstreamHttpConfig,
+};