Right-size sandbox to the worker pod's CPU/memory

cmd/atecontroller/internal/controllers/workerpool_apply.go

@@ -16,11 +16,13 @@ package controllers
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	apiresource "k8s.io/apimachinery/pkg/api/resource"
 	appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1"
 	corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
 	metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
 
 	"github.com/agent-substrate/substrate/internal/ateompath"
+	"github.com/agent-substrate/substrate/internal/sizing"
 	atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
 )
 
@@ -163,6 +165,12 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin
 func ateomContainerEnv(otel ateomOTelSettings) []*corev1ac.EnvVarApplyConfiguration {
 	envs := []*corev1ac.EnvVarApplyConfiguration{
 		fieldRefEnv("POD_UID", "metadata.uid"),
+		// Right-sizing: ateom reads these to size the sandbox (gVisor cgroup leaf /
+		// micro-VM VmConfig) to the worker pod's own limits; see internal/sizing.
+		// resourceFieldRef falls back to node allocatable when no limit is set, so
+		// WorkerPool templates should set CPU/memory limits.
+		resourceFieldRefEnv(sizing.EnvCPUMilli, "ateom", "limits.cpu", "1m"),
+		resourceFieldRefEnv(sizing.EnvMemoryBytes, "ateom", "limits.memory", "1"),
 	}
 	if otel.Endpoint == "" {
 		return envs
@@ -194,6 +202,19 @@ func fieldRefEnv(name, fieldPath string) *corev1ac.EnvVarApplyConfiguration {
 				WithFieldPath(fieldPath)))
 }
 
+// resourceFieldRefEnv builds a downward-API env var sourced from a container's
+// resource field (e.g. limits.cpu) with the given divisor, letting Kubernetes do
+// the unit conversion and round-up.
+func resourceFieldRefEnv(name, container, resource, divisor string) *corev1ac.EnvVarApplyConfiguration {
+	return corev1ac.EnvVar().
+		WithName(name).
+		WithValueFrom(corev1ac.EnvVarSource().
+			WithResourceFieldRef(corev1ac.ResourceFieldSelector().
+				WithContainerName(container).
+				WithResource(resource).
+				WithDivisor(apiresource.MustParse(divisor))))
+}
+
 // ateomGvisorCapabilities is the capability set an unprivileged gVisor worker
 // needs. runsc's gofer maps a full-range identity in a user namespace
 // (SETUID/SETGID/SETPCAP/SETFCAP), the sandbox pivots root and traces the

cmd/atecontroller/internal/controllers/workerpool_apply_test.go

@@ -26,6 +26,7 @@ import (
 	metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
 
 	"github.com/agent-substrate/substrate/internal/ateompath"
+	"github.com/agent-substrate/substrate/internal/sizing"
 	atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
 )
 
@@ -545,11 +546,27 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf
 					WithAdd(ateomGvisorCapabilities...)).
 				WithAppArmorProfile(corev1ac.AppArmorProfile().
 					WithType(corev1.AppArmorProfileTypeUnconfined))).
-			WithEnv(corev1ac.EnvVar().
-				WithName("POD_UID").
-				WithValueFrom(corev1ac.EnvVarSource().
-					WithFieldRef(corev1ac.ObjectFieldSelector().
-						WithFieldPath("metadata.uid")))).
+			WithEnv(
+				corev1ac.EnvVar().
+					WithName("POD_UID").
+					WithValueFrom(corev1ac.EnvVarSource().
+						WithFieldRef(corev1ac.ObjectFieldSelector().
+							WithFieldPath("metadata.uid"))),
+				corev1ac.EnvVar().
+					WithName(sizing.EnvCPUMilli).
+					WithValueFrom(corev1ac.EnvVarSource().
+						WithResourceFieldRef(corev1ac.ResourceFieldSelector().
+							WithContainerName("ateom").
+							WithResource("limits.cpu").
+							WithDivisor(resource.MustParse("1m")))),
+				corev1ac.EnvVar().
+					WithName(sizing.EnvMemoryBytes).
+					WithValueFrom(corev1ac.EnvVarSource().
+						WithResourceFieldRef(corev1ac.ResourceFieldSelector().
+							WithContainerName("ateom").
+							WithResource("limits.memory").
+							WithDivisor(resource.MustParse("1")))),
+			).
 			WithVolumeMounts(
 				corev1ac.VolumeMount().
 					WithName("run-ateom").

cmd/ateom-gvisor/runsc.go

@@ -29,6 +29,7 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 
 	"github.com/agent-substrate/substrate/internal/ateompath"
+	"github.com/agent-substrate/substrate/internal/sizing"
 )
 
 type runsc struct {
@@ -57,10 +58,13 @@ func (r *runsc) ensureContainerCgroupsPath(containerName string) error {
 	if spec.Linux == nil {
 		spec.Linux = &specs.Linux{}
 	}
-	if spec.Linux.CgroupsPath != "" {
-		return nil
+	if spec.Linux.CgroupsPath == "" {
+		spec.Linux.CgroupsPath = "/" + containerName
 	}
-	spec.Linux.CgroupsPath = "/" + containerName
+	// Right-size the per-container cgroup leaf to the worker pod's limits; runsc
+	// applies spec.Linux.Resources when it creates the leaf. Shared with the
+	// micro-VM runtime via internal/sizing.
+	sizing.Discover().ApplyToOCISpec(&spec)
 	out, err := json.MarshalIndent(&spec, "", "  ")
 	if err != nil {
 		return fmt.Errorf("marshaling %q: %w", specPath, err)
@@ -90,6 +94,10 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri
 		// "-log-packets",
 		// "-strace",
 		"-root", ateompath.RunSCStateDir(r.actorUID),
+		// Provision the sentry's vCPU count from the cgroup CPU quota written by
+		// sizing.ApplyToOCISpec, so the sandbox is sized to the pod's limit (runsc
+		// otherwise sizes to all host CPUs). Global flag: before the subcommand.
+		"--cpu-num-from-quota",
 		"create",
 		"-bundle", ateompath.OCIBundlePath(r.actorUID, containerName),
 		"-pid-file", ateompath.PIDFilePath(r.actorUID, containerName),
@@ -237,6 +245,8 @@ func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, ch
 		// "-log-packets",
 		// "-strace",
 		"-root", ateompath.RunSCStateDir(r.actorUID),
+		// Match cmdCreate: size the restored sentry from the cgroup CPU quota.
+		"--cpu-num-from-quota",
 		"restore",
 		"-bundle", ateompath.OCIBundlePath(r.actorUID, containerName),
 		"-image-path", checkpointPath,

cmd/ateom-microvm/main.go

@@ -53,11 +53,12 @@ import (
 )
 
 var (
-	podUID      = flag.String("pod-uid", "", "The UID of the current pod")
-	chBinary    = flag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).")
-	kataConfig  = flag.String("kata-config", "", "Path to a kata configuration.toml (passed to the shim as KATA_CONF_FILE). Empty uses kata's default. atelet generates one pointing at runtime-fetched assets.")
-	kataDebug   = flag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.")
-	showVersion = flag.Bool("version", false, "Print version and exit.")
+	podUID        = flag.String("pod-uid", "", "The UID of the current pod")
+	chBinary      = flag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).")
+	kataConfig    = flag.String("kata-config", "", "Path to a kata configuration.toml (passed to the shim as KATA_CONF_FILE). Empty uses kata's default. atelet generates one pointing at runtime-fetched assets.")
+	kataDebug     = flag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.")
+	vmmMemReserve = flag.Int("vmm-mem-reserve-mib", vmmMemReserveMiB, "Guest RAM (MiB) held back from the pod's memory limit for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the pod cgroup alongside the guest RAM. Prevents the pod OOMing when the VM is sized to the pod's memory limit.")
+	showVersion   = flag.Bool("version", false, "Print version and exit.")
 
 	atunnelListenAddress       = flag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS")
 	atunnelCredentialBundle    = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "PEM credential bundle for actor ingress HTTPS")
@@ -208,7 +209,7 @@ func do(ctx context.Context) error {
 		grpc.StatsHandler(otelgrpc.NewServerHandler()),
 		grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor),
 	)
-	ateompb.RegisterAteomServer(svr, NewService(*podUID, *chBinary, *kataConfig, *kataDebug, interiorNetNS, actorLogger, atunnelServer, atunnelEgress, atunnelEgressPort, *atunnelCredentialBundle, *atunnelEgressTrustBundle))
+	ateompb.RegisterAteomServer(svr, NewService(*podUID, *chBinary, *kataConfig, *kataDebug, *vmmMemReserve, interiorNetNS, actorLogger, atunnelServer, atunnelEgress, atunnelEgressPort, *atunnelCredentialBundle, *atunnelEgressTrustBundle))
 	reflection.Register(svr)
 
 	slog.InfoContext(ctx, "ateom-microvm serving", slog.String("socket", sockPath))
@@ -259,6 +260,11 @@ type AteomService struct {
 	kataConfig string
 	kataDebug  bool
 
+	// memReserveMiB is guest RAM (MiB) held back from the pod's memory limit for
+	// the cloud-hypervisor VMM + virtiofsd (host processes sharing the pod cgroup
+	// with the guest RAM). Set from --vmm-mem-reserve-mib.
+	memReserveMiB int
+
 	// interiorNetNS hosts the per-activation actor veth peer (see net.go);
 	// kata is pointed at it.
 	interiorNetNS netns.NsHandle
@@ -284,12 +290,13 @@ type AteomService struct {
 var _ ateompb.AteomServer = (*AteomService)(nil)
 
 // NewService creates a new AteomService.
-func NewService(podUID, chBinary, kataConfig string, kataDebug bool, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelServer *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, credentialBundle, egressTrustBundle string) *AteomService {
+func NewService(podUID, chBinary, kataConfig string, kataDebug bool, memReserveMiB int, interiorNetNS netns.NsHandle, actorLogger *actorlog.ActorLogger, atunnelServer *atunnel.Server, atunnelEgress *atunnel.Egress, atunnelEgressPort uint16, credentialBundle, egressTrustBundle string) *AteomService {
 	return &AteomService{
 		podUID:                   podUID,
 		chBinary:                 chBinary,
 		kataConfig:               kataConfig,
 		kataDebug:                kataDebug,
+		memReserveMiB:            memReserveMiB,
 		interiorNetNS:            interiorNetNS,
 		actorLogger:              actorLogger,
 		atunnel:                  atunnelServer,

cmd/ateom-microvm/run.go

@@ -38,6 +38,7 @@ import (
 	"github.com/agent-substrate/substrate/internal/proto/ateompb"
 	"github.com/agent-substrate/substrate/internal/readyz"
 	"github.com/agent-substrate/substrate/internal/resources"
+	"github.com/agent-substrate/substrate/internal/sizing"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc/codes"
@@ -105,6 +106,12 @@ const (
 	assetVirtiofsd = "virtiofsd"
 )
 
+// vmmMemReserveMiB is the DEFAULT guest RAM held back from the pod's memory limit
+// for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the same
+// pod cgroup as the guest RAM; without a margin the pod OOMs. Overridable per
+// deployment via --vmm-mem-reserve-mib (see AteomService.memReserveMiB).
+const vmmMemReserveMiB = 256
+
 // maxActorContainers is a sanity cap on containers per actor (all share the one
 // micro-VM + virtiofsd). 25 is far above any real pod.
 const maxActorContainers = 25
@@ -335,6 +342,22 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re
 		return err
 	}
 
+	// Right-size the VM to the worker pod's limits (see internal/sizing), keeping
+	// the kata-config values above as the fallback when a limit is unset. vCPUs
+	// round up; VM RAM reserves a fixed margin for the VMM + virtiofsd, which share
+	// the pod cgroup with the guest RAM. NB: a FULL-scope snapshot restore reuses
+	// the size baked into the snapshot (restoreFullScope), so resizing an existing
+	// actor takes effect on its next cold boot.
+	sz := sizing.Discover()
+	if v := sz.VCPUs(); v > 0 {
+		vcpus = v
+	}
+	if sz.MemoryBytes > 0 {
+		if m := int(sz.MemoryBytes/(1024*1024)) - s.memReserveMiB; m > 0 {
+			memMiB = m
+		}
+	}
+
 	// Clean stale per-sandbox state + create the runtime dir for the sockets.
 	kata.CleanupSandboxState(ctx, actorUID)
 	if err := os.MkdirAll(kata.VMDir(actorUID), 0o700); err != nil {

cmd/ateom-microvm/spec.go

@@ -24,6 +24,8 @@ import (
 	"strings"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"
+
+	"github.com/agent-substrate/substrate/internal/sizing"
 )
 
 // ensureKataCompatibleSpec augments the bundle's config.json with the fields
@@ -51,6 +53,11 @@ func ensureKataCompatibleSpec(bundle, id, netnsPath string) (*specs.Spec, error)
 	if spec.Linux.CgroupsPath == "" {
 		spec.Linux.CgroupsPath = "/ateomchv/" + id
 	}
+	// Right-size the guest container cgroup to the worker pod's limits; the
+	// kata-agent applies spec.Linux.Resources inside the VM. Shared with the gVisor
+	// runtime via internal/sizing; overlays the device allowlist + CPU shares set
+	// by defaultKataResources.
+	sizing.Discover().ApplyToOCISpec(&spec)
 
 	// atelet's spec carries gVisor pause-model CRI annotations
 	// (container-type=container, sandbox-id=pause). kata reads those and waits

demos/counter/counter-microvm.yaml.tmpl

@@ -100,6 +100,17 @@ spec:
   sandboxClass: microvm
   sandboxConfigName: counter-microvm
   ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm
+  template:
+    # Per-worker resources; the limits size the micro-VM (vCPUs + RAM, less a
+    # VMM reserve) via the downward API (see internal/sizing). Keep memory well
+    # above the VMM reserve so the guest gets usable RAM.
+    resources:
+      requests:
+        cpu: "1"
+        memory: 1Gi
+      limits:
+        cpu: "2"
+        memory: 2Gi
 
 ---
 

demos/counter/counter.yaml.tmpl

@@ -29,6 +29,15 @@ metadata:
 spec:
   replicas: 5
   ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+  template:
+    # Per-worker resources; the limits size each sandbox (see internal/sizing).
+    resources:
+      requests:
+        cpu: "250m"
+        memory: 256Mi
+      limits:
+        cpu: "1"
+        memory: 1Gi
 
 ---
 

demos/sandbox/sandbox.yaml.tmpl

@@ -27,6 +27,17 @@ metadata:
 spec:
   replicas: 2
   ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+  template:
+    # Per-worker resources. The limits size each sandbox: ateom reads them via
+    # the downward API and applies them to the gVisor cgroup leaf / micro-VM
+    # (see internal/sizing). Set limits so sizing does not fall back to the node.
+    resources:
+      requests:
+        cpu: "500m"
+        memory: 512Mi
+      limits:
+        cpu: "2"
+        memory: 2Gi
 ---
 apiVersion: ate.dev/v1alpha1
 kind: ActorTemplate

docs/api-guide.md

@@ -26,6 +26,19 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage
 | `nodeAffinity` | `NodeAffinity` | `spec.affinity.nodeAffinity` |
 | `resources` | `ResourceRequirements` | `spec.containers[].resources` |
 
+#### Sandbox Right-Sizing (`spec.template.resources`)
+
+Setting `resources.limits` (CPU and Memory) on a `WorkerPool` directly sizes the sandboxes hosted on those worker pods (rather than sizing to the whole host):
+
+- **gVisor (`ateom-gvisor`)**:
+  - `limits.cpu`: Configures the cgroup v2 CPU quota (`cpu.max`) and provisions the Sentry vCPU count (`--cpu-num-from-quota`), ensuring the sandboxed applications see matching CPU cores and do not oversubscribe threads.
+  - `limits.memory`: Sets the cgroup v2 memory limit (`memory.max`) and bounds the virtual total memory seen by the sandbox (preventing applications like JVM/Go from over-allocating based on host RAM).
+- **Micro-VM (`ateom-microvm`)**:
+  - `limits.cpu`: Determines the Cloud Hypervisor `BootVcpus` / `MaxVcpus` (rounded up to whole vCPUs).
+  - `limits.memory`: Sets the guest RAM for the VM, reserving a small configurable margin (default 256 MiB, tunable via `--vmm-mem-reserve-mib`) for the VMM and virtiofsd host processes so the pod cgroup does not OOM.
+- **Default Fallback**:
+  - If `limits` are omitted, the Kubernetes Downward API falls back to the host node's allocatable capacity. Setting explicit `limits` on `WorkerPool.spec.template.resources` is recommended so sandboxes are sized to the pod rather than the whole node.
+
 ### Example
 
 ```yaml