Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions cmd/atenet/internal/router/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

Router has several responsibilities:

* (Optional) manages a Deployment of Envoy to function as a router for ATE requests.
* This is optional to enable testing the router component in a standalone mode without managing the Kubernetes objects.
* Envoy will be configured to send traffic to via xDS served by the Router.
* ext_proc server for the Envoy. To make the deployment and debugging easier, we will run this component together
* Serves Envoy xDS configuration when `--atenet-router=envoy` (the default).
Unless `--standalone` is set, it also manages the Envoy Deployment and
Services in Kubernetes.
With `--atenet-router=agentgateway`, the sidecar uses a static ConfigMap and
atenet does not start an xDS server.
* ext_proc server for the proxy. To make the deployment and debugging easier, we will run this component together
with the router, but this will be split later into its own component.
* ext_proc will call into the ATE gRPC API to get the set of relevant backends (specific the worker IP) and
route the traffic accordingly
Expand All @@ -27,4 +29,4 @@ Contents:
* Global flags values
* Command line args
* Last 100 queries served
* Build tag
* Build tag
1 change: 1 addition & 0 deletions cmd/atenet/internal/router/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func NewRouterCmd() *cobra.Command {
cmd.Flags().StringVar(&cfg.LogLevel, "log-level", "info", "Log level: debug, info, warn, error")
cmd.Flags().StringVar(&cfg.MetricsAddr, "metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.")
cmd.Flags().BoolVar(&cfg.Standalone, "standalone", false, "Run in standalone mode, bypassing creation of managed deployment and services in Kubernetes cluster")
cmd.Flags().StringVar(&cfg.AtenetRouter, "atenet-router", string(atenetRouterEnvoy), "Router dataplane: envoy or agentgateway")
cmd.Flags().StringVar(&cfg.Namespace, "namespace", "default", "Target operations namespace")
cmd.Flags().StringVar(&cfg.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig configuration file")
cmd.Flags().StringVar(&cfg.AteapiAddr, "ateapi-address", "dns:///api.ate-system.svc:443", "gRPC dial target for the cluster ateapi Control instance.")
Expand Down
21 changes: 21 additions & 0 deletions cmd/atenet/internal/router/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ import (
"time"
)

type atenetRouter string

const (
atenetRouterEnvoy atenetRouter = "envoy"
atenetRouterAgentgateway atenetRouter = "agentgateway"
)

// authConfig holds the router's client-auth settings for dialing ateapi.
// AteapiCAFile always verifies ateapi's serving cert (the servicedns trust
// bundle in-cluster). By default the router presents AteapiClientCertPath
Expand All @@ -36,6 +43,7 @@ type authConfig struct {
// routerConfig holds deployment setup and endpoint options for the router node instance.
type routerConfig struct {
Standalone bool
AtenetRouter string
Namespace string
Kubeconfig string
AteapiAddr string
Expand Down Expand Up @@ -86,6 +94,13 @@ type routerConfig struct {
ExtProcMaxRequests int
}

func (c routerConfig) atenetRouter() atenetRouter {
if c.AtenetRouter == "" {
return atenetRouterEnvoy
}
return atenetRouter(c.AtenetRouter)
}

// extProcMaxRequestsFloor is the minimum derived circuit breaker — Envoy's own
// default max_requests — so a small (or disabled) parking lot still leaves
// ordinary fast-path capacity.
Expand All @@ -109,9 +124,15 @@ func (c routerConfig) extProcMaxRequests() int {
// validate rejects flag combinations that would make the router misbehave
// rather than merely differ.
func (c routerConfig) validate() error {
switch c.atenetRouter() {
case atenetRouterEnvoy, atenetRouterAgentgateway:
default:
return fmt.Errorf("--atenet-router must be %q or %q, got %q", atenetRouterEnvoy, atenetRouterAgentgateway, c.AtenetRouter)
}
if err := c.ParkedRequest.validate(); err != nil {
return err
}

if c.ExtProcMaxRequests < 0 {
return fmt.Errorf("--extproc-max-requests must not be negative, got %d (0 derives it from --parked-request-max)", c.ExtProcMaxRequests)
}
Expand Down
34 changes: 33 additions & 1 deletion cmd/atenet/internal/router/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,22 @@ func TestRouterConfigValidate(t *testing.T) {
wantErr string // substring; empty means valid
}{
{
name: "defaults are valid (auto breaker)",
name: "atenet-router defaults to envoy",
cfg: routerConfig{ExtProcMaxRequests: 0, ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}},
},
{
name: "atenet-router set to envoy is valid",
cfg: routerConfig{AtenetRouter: string(atenetRouterEnvoy), ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}},
},
{
name: "atenet-router set to agentgateway is valid",
cfg: routerConfig{AtenetRouter: string(atenetRouterAgentgateway), ParkedRequest: ParkedRequestConfig{Max: defaultParkedRequestMax}},
},
{
name: "unknown router rejected",
cfg: routerConfig{AtenetRouter: "blah"},
wantErr: "--atenet-router must be",
},
{
name: "negative extproc-max-requests rejected",
cfg: routerConfig{ExtProcMaxRequests: -1, ParkedRequest: ParkedRequestConfig{Max: 0}},
Expand Down Expand Up @@ -64,6 +77,25 @@ func TestRouterConfigValidate(t *testing.T) {
}
}

func TestRouterConfigAtenetRouter(t *testing.T) {
tests := []struct {
name string
cfg routerConfig
want atenetRouter
}{
{name: "default", cfg: routerConfig{}, want: atenetRouterEnvoy},
{name: "explicit envoy", cfg: routerConfig{AtenetRouter: string(atenetRouterEnvoy)}, want: atenetRouterEnvoy},
{name: "agentgateway", cfg: routerConfig{AtenetRouter: string(atenetRouterAgentgateway)}, want: atenetRouterAgentgateway},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := tc.cfg.atenetRouter(); got != tc.want {
t.Fatalf("atenetRouter() = %q, want %q", got, tc.want)
}
})
}
}

func TestRouterConfigExtProcMaxRequests(t *testing.T) {
tests := []struct {
name string
Expand Down
12 changes: 6 additions & 6 deletions cmd/atenet/internal/router/dashboard.html
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ <h1>atenet Router Status</h1>
<div class="card">
<div class="card-title">Component Network Allocation</div>
<div class="metadata-item">
<span class="label">Workload Port (Http Envoy)</span>
<span class="label">Workload Port (HTTP Dataplane)</span>
<span class="value">{{ .HttpPort }}</span>
</div>
<div class="metadata-item">
Expand Down Expand Up @@ -298,20 +298,20 @@ <h1>atenet Router Status</h1>
<div class="card-title">System Component Health Checks</div>

<div class="metadata-item">
<span class="label">Envoy Health</span>
{{ if .Health.Envoy.Healthy }}
<span class="label">Dataplane Health</span>
{{ if .Health.Dataplane.Healthy }}
<span class="value badge" style="background: rgba(16, 185, 129, 0.1); color: #10b981;">Healthy</span>
{{ else }}
<span class="value badge" style="background: rgba(239, 68, 68, 0.1); color: #ef4444;">Degraded</span>
{{ end }}
</div>
<div class="metadata-item" style="margin-top: -0.25rem; margin-bottom: 0.5rem; font-size: 0.8rem;">
<span class="label">Message / Err:</span>
<span class="value" style="color: var(--text-secondary);">{{ .Health.Envoy.Message }}</span>
<span class="value" style="color: var(--text-secondary);">{{ .Health.Dataplane.Message }}</span>
</div>
<div class="metadata-item" style="font-size: 0.75rem; color: var(--text-secondary);">
<span>Ok: {{ .Health.Envoy.SuccessCount }} | Err: {{ .Health.Envoy.FailureCount }}</span>
<span>LKG: {{ if not .Health.Envoy.LastSuccess.IsZero }}{{ .Health.Envoy.LastSuccess.Format "15:04:05" }}{{ else }}N/A{{ end }}</span>
<span>Ok: {{ .Health.Dataplane.SuccessCount }} | Err: {{ .Health.Dataplane.FailureCount }}</span>
<span>LKG: {{ if not .Health.Dataplane.LastSuccess.IsZero }}{{ .Health.Dataplane.LastSuccess.Format "15:04:05" }}{{ else }}N/A{{ end }}</span>
</div>

<div style="border-bottom: 1px solid var(--card-border); margin: 0.5rem 0;"></div>
Expand Down
90 changes: 90 additions & 0 deletions cmd/atenet/internal/router/dataplane.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package router

import (
"context"
"fmt"
"log/slog"
"net"
"time"

"golang.org/x/sync/errgroup"
)

type dataplaneHealthCheck struct {
url string
expectedBody string
}

func (r atenetRouter) routeViaAuthority() bool {
return r == atenetRouterAgentgateway
}

func (r atenetRouter) healthCheck() dataplaneHealthCheck {
switch r {
case atenetRouterEnvoy:
return dataplaneHealthCheck{url: "http://127.0.0.1:9901/ready", expectedBody: "LIVE"}
case atenetRouterAgentgateway:
return dataplaneHealthCheck{url: "http://127.0.0.1:15021/healthz/ready", expectedBody: "ready"}
default:
return dataplaneHealthCheck{}
}
}

func (s *RouterServer) startDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig) error {
switch s.cfg.atenetRouter() {
case atenetRouterEnvoy:
s.startEnvoyDataplane(ctx, g, parkCfg)
case atenetRouterAgentgateway:
// Agentgateway receives all routing configuration from its static file.
default:
return fmt.Errorf("unsupported atenet router %q", s.cfg.atenetRouter())
}
return nil
}

func (s *RouterServer) startEnvoyDataplane(ctx context.Context, g *errgroup.Group, parkCfg ParkedRequestConfig) {
xdsSrv := NewXdsServer(s.cfg.XdsPort)
xdsSrv.SetConfig(s.cfg.HttpPort, s.cfg.ExtprocPort, s.cfg.ExtprocAddr)
setOtlpCollector(ctx, xdsSrv, s.cfg.OtlpCollectorAddress)

xdsSrv.SetExtProcMaxRequests(s.cfg.extProcMaxRequests())
if parkCfg.enabled() {
// Envoy must keep a parked request open at least as long as the router
// will hold it; add a margin so the router surfaces its own 503 first.
xdsSrv.SetExtProcMessageTimeout(parkCfg.Budget + 5*time.Second)
}

xdsSrv.SetTlsConfig(s.cfg.HttpsPort, s.cfg.EnvoyCertPath)
xdsSrv.SetUpstreamTls(s.cfg.UpstreamCredentialBundlePath, s.cfg.UpstreamTrustBundlePath, s.cfg.UpstreamSpiffePrefix)
ctrl := NewController(s.k8sClient, s.clientset, s.cfg, xdsSrv, s.extprocSrv)

// Envoy receives all routing configuration from the local xDS server.
g.Go(func() error {
slog.InfoContext(ctx, "Starting ActorTemplate controller")
return ctrl.Start(ctx)
})
g.Go(func() error {
slog.InfoContext(ctx, "Starting Envoy xDS Server", slog.Int("port", s.cfg.XdsPort))
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", s.cfg.XdsPort))
if err != nil {
return fmt.Errorf("failed to listen on port %d: %w", s.cfg.XdsPort, err)
}
defer lis.Close()

return xdsSrv.Serve(ctx, lis)
})
}
30 changes: 16 additions & 14 deletions cmd/atenet/internal/router/extproc.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,22 +40,24 @@ import (
// ExtProcServer implements the Envoy external processing gRPC server
// to dynamically manage actor activations based on request traffic.
type ExtProcServer struct {
port int
apiClient ateapipb.ControlClient
recorder *QueryRecorder
resumer *ActorResumer
routeDuration metric.Float64Histogram
parking *parkingLot
port int
apiClient ateapipb.ControlClient
recorder *QueryRecorder
resumer *ActorResumer
routeDuration metric.Float64Histogram
parking *parkingLot
routeViaAuthority bool
}

func NewExtProcServer(port int, apiClient ateapipb.ControlClient, routeDuration metric.Float64Histogram, parkCfg ParkedRequestConfig, parkMetrics *parkingMetrics) *ExtProcServer {
func NewExtProcServer(port int, apiClient ateapipb.ControlClient, routeDuration metric.Float64Histogram, parkCfg ParkedRequestConfig, parkMetrics *parkingMetrics, routeViaAuthority bool) *ExtProcServer {
return &ExtProcServer{
port: port,
apiClient: apiClient,
recorder: NewQueryRecorder(100),
resumer: NewActorResumer(apiClient, withParking(parkCfg)),
routeDuration: routeDuration,
parking: newParkingLot(parkCfg, parkMetrics),
port: port,
apiClient: apiClient,
recorder: NewQueryRecorder(100),
resumer: NewActorResumer(apiClient, withParking(parkCfg)),
routeDuration: routeDuration,
parking: newParkingLot(parkCfg, parkMetrics),
routeViaAuthority: routeViaAuthority,
}
}

Expand Down Expand Up @@ -202,7 +204,7 @@ func (s *ExtProcServer) handleRequestHeaders(
// dial, without touching :authority — atunnel authorizes the actor by the
// original Host (actor DNS name).
mutation := &extprocv3.HeaderMutation{}
addOriginalDstMutation(targetAddr, mutation)
addRoutingMutations(targetAddr, metadata.host, s.routeViaAuthority, mutation)

return &extprocv3.HeadersResponse{
Response: &extprocv3.CommonResponse{
Expand Down
4 changes: 3 additions & 1 deletion cmd/atenet/internal/router/extproc_in.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
)

const authorityHeader = ":authority"

type requestMetadata struct {
headers map[string]string
path string
Expand All @@ -44,7 +46,7 @@ func newRequestMetadata(headers []*corev3.HeaderValue) *requestMetadata {
if k == ":path" {
path = val
}
if k == ":authority" || k == "host" {
if k == authorityHeader || k == "host" {
host = val
}
}
Expand Down
26 changes: 26 additions & 0 deletions cmd/atenet/internal/router/extproc_out.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package router

import (
"github.com/agent-substrate/substrate/internal/atunnel"
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3"
Expand Down Expand Up @@ -53,6 +54,31 @@ func addOriginalDstMutation(dst string, mut *extproc.HeaderMutation) {
)
}

// addRoutingMutations overwrites all routing metadata derived from the
// control-plane result. Envoy dials OriginalDstHeader while preserving
// :authority. Agentgateway v1.4.1's static dynamic backend instead dials the
// request :authority, so that mode rewrites it to the worker atunnel address.
// OriginalHostHeader lets atunnel restore and authorize the actor authority.
func addRoutingMutations(dst, actorHost string, routeViaAuthority bool, mut *extproc.HeaderMutation) {
addOriginalDstMutation(dst, mut)
mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{
AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD,
Header: &corev3.HeaderValue{
Key: atunnel.OriginalHostHeader,
RawValue: []byte(actorHost),
},
})
if routeViaAuthority {
mut.SetHeaders = append(mut.SetHeaders, &corev3.HeaderValueOption{
AppendAction: corev3.HeaderValueOption_OVERWRITE_IF_EXISTS_OR_ADD,
Header: &corev3.HeaderValue{
Key: authorityHeader,
RawValue: []byte(dst),
},
})
}
}

func immediateResponse(statusCode envoy_type.StatusCode, message string) *extproc.ProcessingResponse {
return &extproc.ProcessingResponse{
Response: &extproc.ProcessingResponse_ImmediateResponse{
Expand Down
Loading
Loading