From dd708c03d3e337c393b347cbe751767f067e9456 Mon Sep 17 00:00:00 2001 From: Arjun Komath Date: Wed, 24 Jun 2026 19:47:57 +1000 Subject: [PATCH] Improve CLI auth logs and reconciliation Amp-Thread-ID: https://ampcode.com/threads/T-019ef67c-3b8f-77ae-a778-76c429d70c35 Co-authored-by: Amp --- .gitignore | 1 + agent/internal/agent/drift.go | 445 +- agent/internal/agent/handlers.go | 12 +- agent/internal/agent/workqueue.go | 4 +- cli/package.json | 10 +- cli/pnpm-lock.yaml | 348 + cli/src/main.ts | 326 +- web/actions/projects.ts | 7 +- .../dashboard/servers/[id]/loading.tsx | 180 + web/app/(dashboard)/layout-client.tsx | 38 +- web/app/api/v1/agent/expected-state/route.ts | 308 +- web/app/api/v1/manifest/logs/route.ts | 75 + web/app/page.tsx | 5 +- web/components/auth/device-approval-page.tsx | 66 +- web/components/auth/sign-in-page.tsx | 34 +- .../dashboard/dashboard-page-skeleton.tsx | 96 + web/components/service/service-canvas.tsx | 16 +- web/components/settings/api-key-settings.tsx | 300 + web/components/settings/global-settings.tsx | 17 +- web/db/schema.ts | 13 +- web/lib/agent-status.ts | 107 +- web/lib/agent/expected-state.ts | 484 ++ web/lib/api-auth.ts | 43 +- web/lib/auth-client.ts | 6 +- web/lib/auth.ts | 8 +- web/lib/autoheal-policy.ts | 84 + web/lib/cli-service.ts | 22 +- web/lib/deployment-status.ts | 42 + .../inngest/functions/migration-workflow.ts | 3 +- web/lib/inngest/functions/rollout-helpers.ts | 2 +- web/lib/inngest/functions/rollout-utils.ts | 3 +- web/lib/inngest/functions/rollout-workflow.ts | 2 +- .../functions/service-deletion-workflow.ts | 3 +- web/lib/victoria-logs.ts | 6 +- web/lib/work-queue.ts | 51 +- web/package.json | 143 +- web/pnpm-lock.yaml | 7318 +++++++++-------- web/pnpm-workspace.yaml | 6 +- web/tests/autoheal-policy.test.ts | 75 + web/tests/expected-state.test.ts | 117 + web/vitest.config.ts | 10 + 41 files changed, 6561 insertions(+), 4275 deletions(-) create mode 100644 cli/pnpm-lock.yaml create mode 100644 web/app/(dashboard)/dashboard/servers/[id]/loading.tsx create mode 100644 web/app/api/v1/manifest/logs/route.ts create mode 100644 web/components/dashboard/dashboard-page-skeleton.tsx create mode 100644 web/components/settings/api-key-settings.tsx create mode 100644 web/lib/agent/expected-state.ts create mode 100644 web/lib/autoheal-policy.ts create mode 100644 web/lib/deployment-status.ts create mode 100644 web/tests/autoheal-policy.test.ts create mode 100644 web/tests/expected-state.test.ts create mode 100644 web/vitest.config.ts diff --git a/.gitignore b/.gitignore index 9cb7e702..6aff4e36 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ # misc .DS_Store +.idea/ *.pem # debug diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index 80c7b0b5..6a192150 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -15,6 +15,32 @@ import ( "techulus/cloud-agent/internal/wireguard" ) +type reconcileActionKind string + +const ( + actionStopOrphanNoDeploymentID reconcileActionKind = "stop_orphan_no_deployment_id" + actionRemoveOrphanNoDeploymentID reconcileActionKind = "remove_orphan_no_deployment_id" + actionStopUnexpectedContainer reconcileActionKind = "stop_unexpected_container" + actionRemoveUnexpectedContainer reconcileActionKind = "remove_unexpected_container" + actionDeployMissingContainer reconcileActionKind = "deploy_missing_container" + actionStartContainer reconcileActionKind = "start_container" + actionRedeployContainer reconcileActionKind = "redeploy_container" + actionUpdateDNS reconcileActionKind = "update_dns" + actionUpdateTraefik reconcileActionKind = "update_traefik" + actionUpdateCertificates reconcileActionKind = "update_certificates" + actionWriteChallengeRoute reconcileActionKind = "write_challenge_route" + actionUpdateWireGuard reconcileActionKind = "update_wireguard" + actionStartWireGuard reconcileActionKind = "start_wireguard" +) + +type reconcileAction struct { + Kind reconcileActionKind + Description string + DeploymentID string + Expected *agenthttp.ExpectedContainer + Actual *container.Container +} + func (a *Agent) Tick() { switch a.GetState() { case StateIdle: @@ -60,7 +86,7 @@ func (a *Agent) transitionToIdle() { a.SetState(StateIdle) if a.consumeExpectedStateRefresh() { log.Printf("[processing] fetching latest expected state after pending refresh") - // A deploy wake can arrive while processing a previous snapshot. Run one + // A reconcile wake can arrive while processing a previous snapshot. Run one // immediate idle pass after processing to pick up the latest expected state. a.handleIdle() } @@ -85,11 +111,11 @@ func (a *Agent) handleIdle() { a.updateDnsInSync(expected, actual) - changes := a.detectChanges(expected, actual) - if len(changes) > 0 { - log.Printf("[idle] drift detected, %d change(s) to apply:", len(changes)) - for _, change := range changes { - log.Printf(" → %s", change) + actions := a.planReconcile(expected, actual) + if len(actions) > 0 { + log.Printf("[idle] drift detected, %d change(s) to apply:", len(actions)) + for _, action := range actions { + log.Printf(" → %s", action.Description) } log.Printf("[idle] transitioning to PROCESSING") a.expectedState = expected @@ -97,7 +123,6 @@ func (a *Agent) handleIdle() { a.SetState(StateProcessing) return } - } func (a *Agent) handleProcessing() { @@ -115,15 +140,15 @@ func (a *Agent) handleProcessing() { } a.updateDnsInSync(a.expectedState, actual) + actions := a.planReconcile(a.expectedState, actual) - if len(a.detectChanges(a.expectedState, actual)) == 0 { + if len(actions) == 0 { log.Printf("[processing] state converged, transitioning to IDLE") a.transitionToIdle() return } - err = a.reconcileOne(actual) - if err != nil { + if err := a.applyReconcileAction(actions[0]); err != nil { log.Printf("[processing] reconciliation failed: %v, transitioning to IDLE", err) a.transitionToIdle() return @@ -165,8 +190,8 @@ func (a *Agent) getActualState() (*ActualState, error) { return state, nil } -func (a *Agent) detectChanges(expected *agenthttp.ExpectedState, actual *ActualState) []string { - var changes []string +func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualState) []reconcileAction { + var actions []reconcileAction expectedMap := make(map[string]agenthttp.ExpectedContainer) for _, c := range expected.Containers { @@ -180,32 +205,86 @@ func (a *Agent) detectChanges(expected *agenthttp.ExpectedState, actual *ActualS } } - for _, c := range actual.Containers { - if c.DeploymentID == "" { - changes = append(changes, fmt.Sprintf("STOP orphan container %s (no deployment ID)", c.Name)) + for i := range actual.Containers { + act := &actual.Containers[i] + if act.DeploymentID == "" { + if act.State == "running" { + actions = append(actions, reconcileAction{ + Kind: actionStopOrphanNoDeploymentID, + Description: fmt.Sprintf("STOP orphan container %s (no deployment ID)", act.Name), + Actual: act, + }) + } else { + actions = append(actions, reconcileAction{ + Kind: actionRemoveOrphanNoDeploymentID, + Description: fmt.Sprintf("REMOVE orphan container %s (no deployment ID)", act.Name), + Actual: act, + }) + } } } for id, act := range actualMap { if _, exists := expectedMap[id]; !exists { - changes = append(changes, fmt.Sprintf("STOP orphan container %s (deployment %s not in expected state)", act.Name, id[:8])) + actualContainer := act + if act.State == "running" { + actions = append(actions, reconcileAction{ + Kind: actionStopUnexpectedContainer, + Description: fmt.Sprintf("STOP orphan container %s (deployment %s not in expected state)", act.Name, id[:8]), + DeploymentID: id, + Actual: &actualContainer, + }) + } else { + actions = append(actions, reconcileAction{ + Kind: actionRemoveUnexpectedContainer, + Description: fmt.Sprintf("REMOVE orphan container %s (deployment %s not in expected state)", act.Name, id[:8]), + DeploymentID: id, + Actual: &actualContainer, + }) + } } } for id, exp := range expectedMap { if _, exists := actualMap[id]; !exists { - changes = append(changes, fmt.Sprintf("DEPLOY %s (%s)", exp.Name, exp.Image)) + expectedContainer := exp + actions = append(actions, reconcileAction{ + Kind: actionDeployMissingContainer, + Description: fmt.Sprintf("DEPLOY %s (%s)", exp.Name, exp.Image), + DeploymentID: id, + Expected: &expectedContainer, + }) } } for id, exp := range expectedMap { if act, exists := actualMap[id]; exists { + expectedContainer := exp + actualContainer := act if act.State == "created" || act.State == "exited" { - changes = append(changes, fmt.Sprintf("START %s (state: %s)", exp.Name, act.State)) + actions = append(actions, reconcileAction{ + Kind: actionStartContainer, + Description: fmt.Sprintf("START %s (state: %s)", exp.Name, act.State), + DeploymentID: id, + Expected: &expectedContainer, + Actual: &actualContainer, + }) } else if act.State != "running" { - changes = append(changes, fmt.Sprintf("RESTART %s (state: %s)", exp.Name, act.State)) + actions = append(actions, reconcileAction{ + Kind: actionRedeployContainer, + Description: fmt.Sprintf("REDEPLOY %s (state: %s)", exp.Name, act.State), + DeploymentID: id, + Expected: &expectedContainer, + Actual: &actualContainer, + }) } else if normalizeImage(exp.Image) != normalizeImage(act.Image) { - changes = append(changes, fmt.Sprintf("REDEPLOY %s (image: %s → %s)", exp.Name, act.Image, exp.Image)) + actions = append(actions, reconcileAction{ + Kind: actionRedeployContainer, + Description: fmt.Sprintf("REDEPLOY %s (image: %s → %s)", exp.Name, act.Image, exp.Image), + DeploymentID: id, + Expected: &expectedContainer, + Actual: &actualContainer, + }) } } } @@ -217,7 +296,10 @@ func (a *Agent) detectChanges(expected *agenthttp.ExpectedState, actual *ActualS } expectedDnsHash := dns.HashRecords(expectedDnsRecords) if expectedDnsHash != actual.DnsConfigHash { - changes = append(changes, fmt.Sprintf("UPDATE DNS (%d records)", len(expected.Dns.Records))) + actions = append(actions, reconcileAction{ + Kind: actionUpdateDNS, + Description: fmt.Sprintf("UPDATE DNS (%d records)", len(expected.Dns.Records)), + }) } } @@ -225,14 +307,20 @@ func (a *Agent) detectChanges(expected *agenthttp.ExpectedState, actual *ActualS expectedHttpRoutes := ConvertToHttpRoutes(expected.Traefik.HttpRoutes) expectedTraefikHash := traefik.HashRoutesWithServerName(expectedHttpRoutes, expected.ServerName) if expectedTraefikHash != actual.TraefikConfigHash { - changes = append(changes, fmt.Sprintf("UPDATE Traefik HTTP (%d routes)", len(expected.Traefik.HttpRoutes))) + actions = append(actions, reconcileAction{ + Kind: actionUpdateTraefik, + Description: fmt.Sprintf("UPDATE Traefik HTTP (%d routes)", len(expected.Traefik.HttpRoutes)), + }) } tcpRoutes := ConvertToTCPRoutes(expected.Traefik.TCPRoutes) udpRoutes := ConvertToUDPRoutes(expected.Traefik.UDPRoutes) expectedL4Hash := traefik.HashTCPRoutes(tcpRoutes) + traefik.HashUDPRoutes(udpRoutes) if expectedL4Hash != actual.L4ConfigHash { - changes = append(changes, fmt.Sprintf("UPDATE Traefik L4 (%d TCP, %d UDP routes)", len(tcpRoutes), len(udpRoutes))) + actions = append(actions, reconcileAction{ + Kind: actionUpdateTraefik, + Description: fmt.Sprintf("UPDATE Traefik L4 (%d TCP, %d UDP)", len(tcpRoutes), len(udpRoutes)), + }) } expectedCerts := make([]traefik.Certificate, len(expected.Traefik.Certificates)) @@ -241,11 +329,17 @@ func (a *Agent) detectChanges(expected *agenthttp.ExpectedState, actual *ActualS } expectedCertsHash := traefik.HashCertificates(expectedCerts) if expectedCertsHash != actual.CertificatesHash { - changes = append(changes, fmt.Sprintf("UPDATE Certificates (%d certs)", len(expected.Traefik.Certificates))) + actions = append(actions, reconcileAction{ + Kind: actionUpdateCertificates, + Description: fmt.Sprintf("UPDATE Certificates (%d certs)", len(expected.Traefik.Certificates)), + }) } if expected.Traefik.ChallengeRoute != nil && !actual.ChallengeRouteWritten { - changes = append(changes, "WRITE Challenge Route") + actions = append(actions, reconcileAction{ + Kind: actionWriteChallengeRoute, + Description: "WRITE Challenge Route", + }) } } @@ -257,12 +351,21 @@ func (a *Agent) detectChanges(expected *agenthttp.ExpectedState, actual *ActualS Endpoint: p.Endpoint, } } - expectedWgHash := wireguard.HashPeers(expectedWgPeers) - if expectedWgHash != actual.WireguardHash { - changes = append(changes, fmt.Sprintf("UPDATE WireGuard (%d peers)", len(expected.Wireguard.Peers))) + if wireguard.HashPeers(expectedWgPeers) != actual.WireguardHash { + actions = append(actions, reconcileAction{ + Kind: actionUpdateWireGuard, + Description: fmt.Sprintf("UPDATE WireGuard (%d peers)", len(expected.Wireguard.Peers)), + }) + } + + if !wireguard.IsUp(wireguard.DefaultInterface) { + actions = append(actions, reconcileAction{ + Kind: actionStartWireGuard, + Description: "START WireGuard", + }) } - return changes + return actions } func normalizeImage(image string) string { @@ -283,204 +386,172 @@ func normalizeImage(image string) string { return image + digest } -func (a *Agent) reconcileOne(actual *ActualState) error { - expectedMap := make(map[string]agenthttp.ExpectedContainer) - for _, c := range a.expectedState.Containers { - expectedMap[c.DeploymentID] = c - } +func (a *Agent) applyReconcileAction(action reconcileAction) error { + log.Printf("[reconcile] %s", action.Description) - actualMap := make(map[string]container.Container) - for _, c := range actual.Containers { - if c.DeploymentID != "" { - actualMap[c.DeploymentID] = c + switch action.Kind { + case actionStopOrphanNoDeploymentID, actionStopUnexpectedContainer: + if action.Actual == nil { + return fmt.Errorf("missing actual container for %s", action.Kind) } - } + if err := container.Stop(action.Actual.ID); err != nil { + return fmt.Errorf("failed to stop orphan container: %w", err) + } + return nil - for _, act := range actual.Containers { - if act.DeploymentID == "" { - if act.State == "running" { - log.Printf("[reconcile] stopping orphan container %s (no deployment ID)", act.ID) - if err := container.Stop(act.ID); err != nil { - return fmt.Errorf("failed to stop orphan container: %w", err) - } - return nil - } else { - log.Printf("[reconcile] removing orphan container %s (no deployment ID)", act.ID) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - err := retry.WithBackoff(ctx, retry.ForceRemoveBackoff, func() (bool, error) { - if err := container.ForceRemove(act.ID); err != nil { - log.Printf("[reconcile] remove attempt failed: %v, retrying...", err) - return false, err - } - return true, nil - }) - cancel() - if err != nil { - log.Printf("[reconcile] warning: failed to remove orphan container after retries: %v", err) - } - return nil + case actionRemoveOrphanNoDeploymentID: + if action.Actual == nil { + return fmt.Errorf("missing actual container for %s", action.Kind) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + err := retry.WithBackoff(ctx, retry.ForceRemoveBackoff, func() (bool, error) { + if err := container.ForceRemove(action.Actual.ID); err != nil { + log.Printf("[reconcile] remove attempt failed: %v, retrying...", err) + return false, err } + return true, nil + }) + cancel() + if err != nil { + log.Printf("[reconcile] warning: failed to remove orphan container after retries: %v", err) } - } + return nil - for id, act := range actualMap { - if _, exists := expectedMap[id]; !exists { - if act.State == "running" { - log.Printf("[reconcile] stopping orphan container %s (deployment %s not in expected state)", act.Name, id[:8]) - if err := container.Stop(act.ID); err != nil { - return fmt.Errorf("failed to stop orphan container: %w", err) - } - return nil - } else { - log.Printf("[reconcile] removing orphan container %s (deployment %s not in expected state)", act.Name, id[:8]) - if err := container.ForceRemove(act.ID); err != nil { - log.Printf("[reconcile] warning: failed to remove orphan: %v", err) - } - return nil - } + case actionRemoveUnexpectedContainer: + if action.Actual == nil { + return fmt.Errorf("missing actual container for %s", action.Kind) } - } + if err := container.ForceRemove(action.Actual.ID); err != nil { + log.Printf("[reconcile] warning: failed to remove orphan: %v", err) + } + return nil - for id, exp := range expectedMap { - if _, exists := actualMap[id]; !exists { - log.Printf("[reconcile] deploying missing container for deployment %s", id) - if err := a.Reconciler.Deploy(exp); err != nil { - return fmt.Errorf("failed to deploy container: %w", err) - } - return nil + case actionDeployMissingContainer: + if action.Expected == nil { + return fmt.Errorf("missing expected container for %s", action.Kind) } - } + if err := a.Reconciler.Deploy(*action.Expected); err != nil { + return fmt.Errorf("failed to deploy container: %w", err) + } + return nil - for id, exp := range expectedMap { - if act, exists := actualMap[id]; exists { - if act.State == "created" || act.State == "exited" { - log.Printf("[reconcile] starting %s container %s for deployment %s", act.State, act.ID, id) - if err := container.Start(act.ID); err != nil { - log.Printf("[reconcile] start failed, will redeploy: %v", err) - if err := container.Stop(act.ID); err != nil { - log.Printf("[reconcile] warning: failed to stop old container: %v", err) - } - if err := a.Reconciler.Deploy(exp); err != nil { - return fmt.Errorf("failed to redeploy container: %w", err) - } - } - return nil + case actionStartContainer: + if action.Actual == nil || action.Expected == nil { + return fmt.Errorf("missing container state for %s", action.Kind) + } + if err := container.Start(action.Actual.ID); err != nil { + log.Printf("[reconcile] start failed, will redeploy: %v", err) + if err := container.Stop(action.Actual.ID); err != nil { + log.Printf("[reconcile] warning: failed to stop old container: %v", err) } - if act.State != "running" || normalizeImage(exp.Image) != normalizeImage(act.Image) { - log.Printf("[reconcile] redeploying container for deployment %s (state=%s)", id, act.State) - if err := container.Stop(act.ID); err != nil { - log.Printf("[reconcile] warning: failed to stop old container: %v", err) - } - if err := a.Reconciler.Deploy(exp); err != nil { - return fmt.Errorf("failed to redeploy container: %w", err) - } - return nil + if err := a.Reconciler.Deploy(*action.Expected); err != nil { + return fmt.Errorf("failed to redeploy container: %w", err) } } - } + return nil - if !a.DisableDNS { + case actionRedeployContainer: + if action.Actual == nil || action.Expected == nil { + return fmt.Errorf("missing container state for %s", action.Kind) + } + if err := container.Stop(action.Actual.ID); err != nil { + log.Printf("[reconcile] warning: failed to stop old container: %v", err) + } + if err := a.Reconciler.Deploy(*action.Expected); err != nil { + return fmt.Errorf("failed to redeploy container: %w", err) + } + return nil + + case actionUpdateDNS: expectedDnsRecords := make([]dns.DnsRecord, len(a.expectedState.Dns.Records)) for i, r := range a.expectedState.Dns.Records { expectedDnsRecords[i] = dns.DnsRecord{Name: r.Name, Ips: r.Ips} } - if dns.HashRecords(expectedDnsRecords) != actual.DnsConfigHash { - log.Printf("[reconcile] updating DNS records") - if err := dns.UpdateDnsRecords(expectedDnsRecords); err != nil { - return fmt.Errorf("failed to update DNS: %w", err) - } - return nil + if err := dns.UpdateDnsRecords(expectedDnsRecords); err != nil { + return fmt.Errorf("failed to update DNS: %w", err) } - } - - if a.IsProxy { - expectedHttpRoutes := ConvertToHttpRoutes(a.expectedState.Traefik.HttpRoutes) - tcpRoutes := ConvertToTCPRoutes(a.expectedState.Traefik.TCPRoutes) - udpRoutes := ConvertToUDPRoutes(a.expectedState.Traefik.UDPRoutes) - - httpDrift := traefik.HashRoutesWithServerName(expectedHttpRoutes, a.expectedState.ServerName) != actual.TraefikConfigHash - expectedL4Hash := traefik.HashTCPRoutes(tcpRoutes) + traefik.HashUDPRoutes(udpRoutes) - l4Drift := expectedL4Hash != actual.L4ConfigHash - - if httpDrift || l4Drift { - var tcpPorts, udpPorts []int - for _, r := range tcpRoutes { - tcpPorts = append(tcpPorts, r.ExternalPort) - } - for _, r := range udpRoutes { - udpPorts = append(udpPorts, r.ExternalPort) - } - - needsRestart := false - if len(tcpPorts) > 0 || len(udpPorts) > 0 { - log.Printf("[reconcile] ensuring L4 entry points: %d TCP, %d UDP", len(tcpPorts), len(udpPorts)) - var err error - needsRestart, err = traefik.EnsureEntryPoints(tcpPorts, udpPorts) - if err != nil { - return fmt.Errorf("failed to ensure entry points: %w", err) - } - } - - log.Printf("[reconcile] updating Traefik routes (HTTP: %d, TCP: %d, UDP: %d)", len(expectedHttpRoutes), len(tcpRoutes), len(udpRoutes)) - if err := traefik.UpdateHttpRoutesWithL4(expectedHttpRoutes, tcpRoutes, udpRoutes, a.expectedState.ServerName); err != nil { - return fmt.Errorf("failed to update Traefik: %w", err) - } + return nil - if needsRestart { - log.Printf("[reconcile] restarting Traefik to apply new entry points") - if err := traefik.ReloadTraefik(); err != nil { - return fmt.Errorf("failed to restart Traefik: %w", err) - } - } - return nil - } + case actionUpdateTraefik: + return a.updateTraefik() + case actionUpdateCertificates: expectedCerts := make([]traefik.Certificate, len(a.expectedState.Traefik.Certificates)) for i, c := range a.expectedState.Traefik.Certificates { expectedCerts[i] = traefik.Certificate{Domain: c.Domain, Certificate: c.Certificate, CertificateKey: c.CertificateKey} } - if traefik.HashCertificates(expectedCerts) != actual.CertificatesHash { - log.Printf("[reconcile] updating certificates") - if err := traefik.UpdateCertificates(expectedCerts); err != nil { - return fmt.Errorf("failed to update certificates: %w", err) - } - return nil + if err := traefik.UpdateCertificates(expectedCerts); err != nil { + return fmt.Errorf("failed to update certificates: %w", err) } + return nil - if a.expectedState.Traefik.ChallengeRoute != nil && !actual.ChallengeRouteWritten { - log.Printf("[reconcile] writing challenge route") - if err := traefik.WriteChallengeRoute(a.expectedState.Traefik.ChallengeRoute.ControlPlaneUrl); err != nil { - return fmt.Errorf("failed to write challenge route: %w", err) - } + case actionWriteChallengeRoute: + if a.expectedState.Traefik.ChallengeRoute == nil { return nil } - } - - expectedWgPeers := make([]wireguard.Peer, len(a.expectedState.Wireguard.Peers)) - for i, p := range a.expectedState.Wireguard.Peers { - expectedWgPeers[i] = wireguard.Peer{ - PublicKey: p.PublicKey, - AllowedIPs: p.AllowedIPs, - Endpoint: p.Endpoint, + if err := traefik.WriteChallengeRoute(a.expectedState.Traefik.ChallengeRoute.ControlPlaneUrl); err != nil { + return fmt.Errorf("failed to write challenge route: %w", err) } - } - wgPeersChanged := wireguard.HashPeers(expectedWgPeers) != actual.WireguardHash - wgIsUp := wireguard.IsUp(wireguard.DefaultInterface) + return nil - if wgPeersChanged { - log.Printf("[reconcile] updating WireGuard peers") + case actionUpdateWireGuard: + expectedWgPeers := make([]wireguard.Peer, len(a.expectedState.Wireguard.Peers)) + for i, p := range a.expectedState.Wireguard.Peers { + expectedWgPeers[i] = wireguard.Peer{ + PublicKey: p.PublicKey, + AllowedIPs: p.AllowedIPs, + Endpoint: p.Endpoint, + } + } if err := a.reconcileWireguard(expectedWgPeers); err != nil { return fmt.Errorf("failed to update WireGuard: %w", err) } return nil - } - if !wgIsUp { - log.Printf("[reconcile] WireGuard interface is down, bringing it up") + case actionStartWireGuard: if err := wireguard.Up(wireguard.DefaultInterface); err != nil { return fmt.Errorf("failed to bring up WireGuard: %w", err) } return nil + + default: + return fmt.Errorf("unknown reconcile action: %s", action.Kind) + } +} + +func (a *Agent) updateTraefik() error { + expectedHttpRoutes := ConvertToHttpRoutes(a.expectedState.Traefik.HttpRoutes) + tcpRoutes := ConvertToTCPRoutes(a.expectedState.Traefik.TCPRoutes) + udpRoutes := ConvertToUDPRoutes(a.expectedState.Traefik.UDPRoutes) + + var tcpPorts, udpPorts []int + for _, r := range tcpRoutes { + tcpPorts = append(tcpPorts, r.ExternalPort) + } + for _, r := range udpRoutes { + udpPorts = append(udpPorts, r.ExternalPort) + } + + needsRestart := false + if len(tcpPorts) > 0 || len(udpPorts) > 0 { + log.Printf("[reconcile] ensuring L4 entry points: %d TCP, %d UDP", len(tcpPorts), len(udpPorts)) + var err error + needsRestart, err = traefik.EnsureEntryPoints(tcpPorts, udpPorts) + if err != nil { + return fmt.Errorf("failed to ensure entry points: %w", err) + } + } + + log.Printf("[reconcile] updating Traefik routes (HTTP: %d, TCP: %d, UDP: %d)", len(expectedHttpRoutes), len(tcpRoutes), len(udpRoutes)) + if err := traefik.UpdateHttpRoutesWithL4(expectedHttpRoutes, tcpRoutes, udpRoutes, a.expectedState.ServerName); err != nil { + return fmt.Errorf("failed to update Traefik: %w", err) + } + + if needsRestart { + log.Printf("[reconcile] restarting Traefik to apply new entry points") + if err := traefik.ReloadTraefik(); err != nil { + return fmt.Errorf("failed to restart Traefik: %w", err) + } } return nil diff --git a/agent/internal/agent/handlers.go b/agent/internal/agent/handlers.go index dad56d9a..5dd68cb0 100644 --- a/agent/internal/agent/handlers.go +++ b/agent/internal/agent/handlers.go @@ -3,6 +3,7 @@ package agent import ( "context" "encoding/json" + "errors" "fmt" "log" "os" @@ -67,16 +68,25 @@ func (a *Agent) ProcessForceCleanup(item agenthttp.WorkQueueItem) error { log.Printf("[force_cleanup] cleaning up %d containers for service %s", len(payload.ContainerIDs), Truncate(payload.ServiceID, 8)) + var cleanupErrors []error for _, containerID := range payload.ContainerIDs { if err := container.Stop(containerID); err != nil { log.Printf("[force_cleanup] failed to stop %s: %v", Truncate(containerID, 12), err) + cleanupErrors = append( + cleanupErrors, + fmt.Errorf("stop %s: %w", Truncate(containerID, 12), err), + ) } if err := container.ForceRemove(containerID); err != nil { log.Printf("[force_cleanup] failed to remove %s: %v", Truncate(containerID, 12), err) + cleanupErrors = append( + cleanupErrors, + fmt.Errorf("remove %s: %w", Truncate(containerID, 12), err), + ) } } - return nil + return errors.Join(cleanupErrors...) } func (a *Agent) ProcessCleanupVolumes(item agenthttp.WorkQueueItem) error { diff --git a/agent/internal/agent/workqueue.go b/agent/internal/agent/workqueue.go index 6aa1690a..8eb02fa0 100644 --- a/agent/internal/agent/workqueue.go +++ b/agent/internal/agent/workqueue.go @@ -113,8 +113,8 @@ func (a *Agent) ProcessWorkItem(item agenthttp.WorkQueueItem) error { return a.ProcessRestart(item) case "stop": return a.ProcessStop(item) - case "deploy": - a.RequestReconcile("deploy work item " + Truncate(item.ID, 8)) + case "deploy", "reconcile": + a.RequestReconcile("reconcile work item " + Truncate(item.ID, 8)) return nil case "force_cleanup": return a.ProcessForceCleanup(item) diff --git a/cli/package.json b/cli/package.json index 1b8a94f8..6cefbb97 100644 --- a/cli/package.json +++ b/cli/package.json @@ -5,11 +5,11 @@ "type": "module", "scripts": { "dev": "node --import tsx src/main.ts", - "build": "bun build src/main.ts --compile --outfile dist/tcloud", - "build:linux-x64": "bun build src/main.ts --compile --target=bun-linux-x64 --outfile dist/tcloud-linux-x64", - "build:linux-arm64": "bun build src/main.ts --compile --target=bun-linux-arm64 --outfile dist/tcloud-linux-arm64", - "build:darwin-x64": "bun build src/main.ts --compile --target=bun-darwin-x64 --outfile dist/tcloud-darwin-x64", - "build:darwin-arm64": "bun build src/main.ts --compile --target=bun-darwin-arm64 --outfile dist/tcloud-darwin-arm64", + "build": "bun build src/main.ts --compile --outfile dist/tc", + "build:linux-x64": "bun build src/main.ts --compile --target=bun-linux-x64 --outfile dist/tc-linux-x64", + "build:linux-arm64": "bun build src/main.ts --compile --target=bun-linux-arm64 --outfile dist/tc-linux-arm64", + "build:darwin-x64": "bun build src/main.ts --compile --target=bun-darwin-x64 --outfile dist/tc-darwin-x64", + "build:darwin-arm64": "bun build src/main.ts --compile --target=bun-darwin-arm64 --outfile dist/tc-darwin-arm64", "typecheck": "tsc --noEmit" }, "dependencies": { diff --git a/cli/pnpm-lock.yaml b/cli/pnpm-lock.yaml new file mode 100644 index 00000000..db827add --- /dev/null +++ b/cli/pnpm-lock.yaml @@ -0,0 +1,348 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + yaml: + specifier: ^2.8.2 + version: 2.9.0 + zod: + specifier: ^4.3.5 + version: 4.4.3 + devDependencies: + '@types/node': + specifier: ^22.17.0 + version: 22.20.0 + tsx: + specifier: ^4.19.2 + version: 4.22.4 + typescript: + specifier: ^5.9.2 + version: 5.9.3 + +packages: + + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@types/node@22.20.0': + resolution: {integrity: sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==} + + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + tsx@4.22.4: + resolution: {integrity: sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + + zod@4.4.3: + resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} + +snapshots: + + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + + '@types/node@22.20.0': + dependencies: + undici-types: 6.21.0 + + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + + fsevents@2.3.3: + optional: true + + tsx@4.22.4: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + undici-types@6.21.0: {} + + yaml@2.9.0: {} + + zod@4.4.3: {} diff --git a/cli/src/main.ts b/cli/src/main.ts index 94814650..61bfca35 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -14,6 +14,8 @@ import { const CLI_VERSION = "0.1.0"; const CLI_CLIENT_ID = "techulus-cli"; +const DEFAULT_LOG_TAIL = 100; +const LOG_POLL_INTERVAL_MS = 2000; type JsonRequestOptions = { method?: string; @@ -21,6 +23,12 @@ type JsonRequestOptions = { body?: unknown; }; +type ErrorResponse = { + error?: string; + message?: string; + code?: string; +}; + type LinkServiceTarget = { id: string; name: string; @@ -43,6 +51,13 @@ type LinkProjectTarget = { environments: LinkEnvironmentTarget[]; }; +type ServiceLog = { + deploymentId: string | undefined; + stream: string; + message: string; + timestamp: string; +}; + function normalizeHost(host: string) { const trimmed = host.trim().replace(/\/$/, ""); if (!trimmed.startsWith("http://") && !trimmed.startsWith("https://")) { @@ -63,13 +78,32 @@ async function requestJson(url: string, options: JsonRequestOptions = {}) { }); const text = await response.text(); - const data = text ? (JSON.parse(text) as T | { error?: string }) : null; + const data = text ? (JSON.parse(text) as T | ErrorResponse) : null; if (!response.ok) { - const message = - data && typeof data === "object" && "error" in data && data.error - ? data.error - : `Request failed with ${response.status}`; + const apiMessage = + data && typeof data === "object" + ? "message" in data && data.message + ? data.message + : "error" in data && data.error + ? data.error + : null + : null; + const code = + data && typeof data === "object" && "code" in data && data.code + ? ` (${data.code})` + : ""; + const message = apiMessage + ? `${apiMessage}${code}` + : `Request failed with ${response.status}`; + + if (response.status === 401 || response.status === 403) { + const host = normalizeHost(new URL(url).origin); + throw new Error( + `${message}\n\nYour CLI session is not authorized. Run:\n tc auth login --host ${host}`, + ); + } + throw new Error(message); } @@ -80,6 +114,35 @@ async function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } +function shortId(id: string) { + if (id.length <= 16) return id; + return `${id.slice(0, 8)}…${id.slice(-4)}`; +} + +function formatStatus(value: string) { + return value.replace(/_/g, " "); +} + +function formatTimestamp(value: string) { + const date = new Date(value); + if (Number.isNaN(date.getTime())) return value; + return date.toISOString(); +} + +function printSection(title: string) { + console.log(`\n${title}`); + console.log("─".repeat(title.length)); +} + +function printField(label: string, value: string | number) { + console.log(` ${label.padEnd(10)} ${value}`); +} + +function printNext(command: string) { + printSection("Next"); + printField("Run", command); +} + function parseOption(args: string[], name: string) { const index = args.indexOf(name); if (index === -1) { @@ -94,16 +157,34 @@ function parseOption(args: string[], name: string) { return value; } +function parseLogLineLimit(args: string[]) { + const rawTail = parseOption(args, "-n") ?? parseOption(args, "--tail"); + if (!rawTail) return null; + + if (!/^\d+$/.test(rawTail)) { + throw new Error("log line count must be a positive integer"); + } + + const tail = Number.parseInt(rawTail, 10); + if (tail < 1 || tail > 1000) { + throw new Error("log line count must be between 1 and 1000"); + } + + return tail; +} + function printUsage() { console.log(`Usage: - tcloud auth login --host - tcloud auth logout - tcloud auth whoami - tcloud init - tcloud link [--force] - tcloud apply - tcloud deploy - tcloud status`); + tc auth login --host + tc auth logout + tc auth whoami + tc init + tc link [--force] + tc apply + tc deploy + tc logs + tc logs -n + tc status`); } async function pathExists(filePath: string) { @@ -140,7 +221,7 @@ async function selectFromList( } if (!process.stdin.isTTY || !process.stdout.isTTY) { - throw new Error("tcloud link requires an interactive terminal."); + throw new Error("tc link requires an interactive terminal."); } const rl = createInterface({ input, output }); @@ -181,7 +262,7 @@ async function ensureManifest(cwd: string) { } catch (error) { if (error instanceof Error && "code" in error && error.code === "ENOENT") { throw new Error( - "No techulus.yml found in the current directory. Run `tcloud init` to create one.", + "No techulus.yml found in the current directory. Run `tc init` to create one.", ); } throw new Error( @@ -201,7 +282,7 @@ function authHeaders(apiKey: string) { async function requireConfig() { const config = await readConfig(); if (!config) { - throw new Error("Not logged in. Run `tcloud auth login --host ` first."); + throw new Error("Not logged in. Run `tc auth login --host ` first."); } return config; @@ -232,9 +313,14 @@ async function commandAuthLogin(args: string[]) { }, }); - console.log(`Visit ${deviceCode.verification_uri}`); - console.log(`Enter code: ${deviceCode.user_code}`); - console.log("Open the verification URL in your browser to continue."); + const verificationUrl = + deviceCode.verification_uri_complete || deviceCode.verification_uri; + + printSection("Device login"); + printField("Host", host); + printField("URL", verificationUrl); + printField("Code", deviceCode.user_code); + console.log("\nOpen the verification URL in your browser to continue."); let accessToken = ""; let intervalMs = deviceCode.interval * 1000; @@ -288,7 +374,7 @@ async function commandAuthLogin(args: string[]) { } } - console.log("\nDevice login approved. Creating a CLI API key..."); + console.log("\n\nDevice approved. Creating a CLI API key..."); const machineName = os.hostname(); const platform = `${process.platform}/${process.arch}`; @@ -317,12 +403,17 @@ async function commandAuthLogin(args: string[]) { user: exchange.user, }); - console.log(`Signed in as ${exchange.user.email}`); + printSection("Signed in"); + printField("User", exchange.user.email); + printField("Name", exchange.user.name); + printField("Host", host); + printField("Key", exchange.keyId ? shortId(exchange.keyId) : "created"); } async function commandAuthLogout() { await deleteConfig(); - console.log("Signed out."); + printSection("Signed out"); + printField("Config", "removed"); } async function commandAuthWhoAmI() { @@ -333,9 +424,10 @@ async function commandAuthWhoAmI() { headers: authHeaders(config.apiKey), }); - console.log(`Signed in as ${whoami.user.email}`); - console.log(`Name: ${whoami.user.name}`); - console.log(`Host: ${config.host}`); + printSection("Account"); + printField("User", whoami.user.email); + printField("Name", whoami.user.name); + printField("Host", config.host); } async function commandInit(cwd: string) { @@ -369,7 +461,9 @@ service: `; await writeFile(manifestPath, manifest, "utf8"); - console.log(`Created ${manifestPath}`); + printSection("Manifest"); + printField("Created", manifestPath); + printNext("tc apply"); } async function commandLink(cwd: string, args: string[]) { @@ -379,7 +473,7 @@ async function commandLink(cwd: string, args: string[]) { if ((await pathExists(manifestPath)) && !force) { throw new Error( - "techulus.yml already exists. Run `tcloud link --force` to replace it.", + "techulus.yml already exists. Run `tc link --force` to replace it.", ); } @@ -459,11 +553,13 @@ async function commandLink(cwd: string, args: string[]) { await writeFile(manifestPath, stringifyManifest(result.manifest), "utf8"); - console.log( - `Linked ${result.service.project}/${result.service.environment}/${result.service.name}`, + printSection("Linked"); + printField( + "Service", + `${result.service.project}/${result.service.environment}/${result.service.name}`, ); - console.log(`Wrote ${manifestPath}`); - console.log("Next: run `tcloud status` or `tcloud apply`."); + printField("Manifest", manifestPath); + printNext("tc status or tc apply"); } function printApplyResult(result: { @@ -471,17 +567,20 @@ function printApplyResult(result: { serviceId: string; changes: Array<{ field: string; from: string; to: string }>; }) { - console.log(`Action: ${result.action}`); - console.log(`Service ID: ${result.serviceId}`); + printSection("Apply"); + printField("Action", result.action); + printField("Service", shortId(result.serviceId)); if (result.changes.length === 0) { - console.log("No changes."); + printField("Changes", "none"); return; } - console.log("Changes:"); + printSection(`Changes (${result.changes.length})`); for (const change of result.changes) { - console.log(`- ${change.field}: ${change.from} -> ${change.to}`); + console.log(` • ${change.field}`); + printField("From", change.from); + printField("To", change.to); } } @@ -514,11 +613,13 @@ async function commandDeploy(cwd: string) { body: manifest, }); - console.log(`Service ID: ${result.serviceId}`); - console.log(`Status: ${result.status}`); + printSection("Deploy"); + printField("Service", shortId(result.serviceId)); + printField("Status", formatStatus(result.status)); if (result.rolloutId) { - console.log(`Rollout ID: ${result.rolloutId}`); + printField("Rollout", shortId(result.rolloutId)); } + printNext("tc status"); } async function commandStatus(cwd: string) { @@ -550,26 +651,148 @@ async function commandStatus(cwd: string) { headers: authHeaders(config.apiKey), }); - console.log(`Service ID: ${status.service.id}`); - console.log(`Image: ${status.service.image}`); - console.log(`Hostname: ${status.service.hostname ?? "(none)"}`); - console.log(`Replicas: ${status.service.replicas}`); + console.log(`${manifest.project}/${manifest.environment}/${manifest.service.name}`); + + printSection("Service"); + printField("ID", shortId(status.service.id)); + printField("Image", status.service.image); + printField("Hostname", status.service.hostname ?? "none"); + printField("Replicas", status.service.replicas); + + printSection("Rollout"); if (status.latestRollout) { - console.log( - `Latest rollout: ${status.latestRollout.id} (${status.latestRollout.status}${status.latestRollout.currentStage ? `, ${status.latestRollout.currentStage}` : ""})`, + printField("ID", shortId(status.latestRollout.id)); + printField("Status", formatStatus(status.latestRollout.status)); + printField( + "Stage", + status.latestRollout.currentStage + ? formatStatus(status.latestRollout.currentStage) + : "none", ); } else { - console.log("Latest rollout: none"); + printField("Latest", "none"); } - console.log(`Deployments: ${status.deployments.length}`); + + printSection(`Deployments (${status.deployments.length})`); + if (status.deployments.length === 0) { + printField("Current", "none"); + return; + } + for (const deployment of status.deployments) { - console.log(`- ${deployment.id}: ${deployment.status} on ${deployment.serverId}`); + console.log(` • ${shortId(deployment.id)}`); + printField("Status", formatStatus(deployment.status)); + printField("Server", shortId(deployment.serverId)); + } +} + +function printLogs(logs: ServiceLog[]) { + for (const log of logs) { + const stream = `[${log.stream || "stdout"}]`.padEnd(9); + const message = log.message.replace(/\n+$/, ""); + console.log(`${formatTimestamp(log.timestamp)} ${stream} ${message}`); + } +} + +function getLogCursor(logs: ServiceLog[]) { + return logs.reduce((latest, log) => { + if (!latest) return log.timestamp; + return new Date(log.timestamp).getTime() > new Date(latest).getTime() + ? log.timestamp + : latest; + }, null); +} + +function getLogKey(log: ServiceLog) { + return `${log.timestamp}:${log.stream}:${log.deploymentId ?? ""}:${log.message}`; +} + +async function fetchManifestLogs( + config: Awaited>, + manifest: TechulusManifest, + options: { tail: number; after?: string | null }, +) { + const params = new URLSearchParams({ + project: manifest.project, + environment: manifest.environment, + service: manifest.service.name, + tail: String(options.tail), + }); + if (options.after) { + params.set("after", options.after); + } + + return requestJson<{ + loggingEnabled: boolean; + logs: ServiceLog[]; + }>(`${config.host}/api/v1/manifest/logs?${params.toString()}`, { + headers: authHeaders(config.apiKey), + }); +} + +async function commandLogs(cwd: string, args: string[]) { + const lineLimit = parseLogLineLimit(args); + const config = await requireConfig(); + const { manifest } = await ensureManifest(cwd); + const result = await fetchManifestLogs(config, manifest, { + tail: lineLimit ?? DEFAULT_LOG_TAIL, + }); + + console.log(`${manifest.project}/${manifest.environment}/${manifest.service.name}`); + + if (!result.loggingEnabled) { + printSection("Logs"); + printField("Status", "disabled"); + return; + } + + if (lineLimit && result.logs.length === 0) { + printSection("Logs"); + printField("Lines", "none"); + return; + } + + if (lineLimit) { + printSection(`Logs (${result.logs.length})`); + printLogs(result.logs); + return; + } + + printSection("Logs"); + if (result.logs.length > 0) { + printLogs(result.logs); + } else { + printField("Waiting", "new log lines"); + } + + let after = getLogCursor(result.logs) ?? new Date().toISOString(); + const seen = new Set(result.logs.map(getLogKey)); + + while (true) { + await sleep(LOG_POLL_INTERVAL_MS); + const next = await fetchManifestLogs(config, manifest, { + tail: DEFAULT_LOG_TAIL, + after, + }); + const logs = next.logs.filter((log) => !seen.has(getLogKey(log))); + if (logs.length === 0) continue; + + printLogs(logs); + for (const log of logs) { + seen.add(getLogKey(log)); + } + after = getLogCursor(logs) ?? after; } } async function main() { - const [command, subcommand, ...rest] = process.argv.slice(2); - const cwd = process.cwd(); + const argv = process.argv.slice(2); + if (argv[0] === "--") { + argv.shift(); + } + + const [command, subcommand, ...rest] = argv; + const cwd = process.env.INIT_CWD || process.cwd(); if (!command) { printUsage(); @@ -604,6 +827,9 @@ async function main() { case "deploy": await commandDeploy(cwd); return; + case "logs": + await commandLogs(cwd, [subcommand, ...rest].filter(Boolean)); + return; case "status": await commandStatus(cwd); return; diff --git a/web/actions/projects.ts b/web/actions/projects.ts index d9820790..b07053ef 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -30,6 +30,7 @@ import { } from "@/db/schema"; import { requireAuth } from "@/lib/auth"; import { DEFAULT_RESOURCE_LIMITS } from "@/lib/constants"; +import { markDeploymentUndesired } from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { allocatePort } from "@/lib/port-allocation"; @@ -512,7 +513,7 @@ async function hardDeleteService(serviceId: string) { ) { await db .update(deployments) - .set({ status: "stopping" }) + .set(markDeploymentUndesired("stopping")) .where(eq(deployments.id, dep.id)); await enqueueWork(dep.serverId, "stop", { @@ -1204,7 +1205,7 @@ export async function stopService(serviceId: string) { await db .update(deployments) - .set({ status: "stopping" }) + .set(markDeploymentUndesired("stopping")) .where(eq(deployments.id, dep.id)); await enqueueWork(dep.serverId, "stop", { @@ -1323,7 +1324,7 @@ export async function abortRollout(serviceId: string) { .where( and( eq(workQueue.status, "pending"), - eq(workQueue.type, "deploy"), + inArray(workQueue.type, ["deploy", "reconcile"]), inArray(workQueue.serverId, [...serverContainers.keys()]), ), ); diff --git a/web/app/(dashboard)/dashboard/servers/[id]/loading.tsx b/web/app/(dashboard)/dashboard/servers/[id]/loading.tsx new file mode 100644 index 00000000..f42210a7 --- /dev/null +++ b/web/app/(dashboard)/dashboard/servers/[id]/loading.tsx @@ -0,0 +1,180 @@ +import { SetBreadcrumbs } from "@/components/core/breadcrumb-data"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; + +function ServerDetailsSkeleton() { + return ( + + + Server Details + + +
+ {Array.from({ length: 9 }).map((_, index) => ( +
+ + +
+ ))} +
+ + + ); +} + +function HealthMetricSkeleton() { + return ( +
+
+ + +
+ + +
+ ); +} + +function HealthStatusSkeleton() { + return ( +
+ +
+ + +
+
+ ); +} + +function SystemHealthSkeleton() { + return ( + + + System Health + + +
+ + + +
+
+
+ + + +
+
+ + + ); +} + +function RunningServicesSkeleton() { + return ( + + + + + + + + + + + +
+ {Array.from({ length: 4 }).map((_, index) => ( +
+ +
+ + +
+
+ ))} +
+ + + ); +} + +function AgentLogsSkeleton() { + return ( +
+

Agent Logs

+
+
+ + +
+
+ {Array.from({ length: 12 }).map((_, index) => ( + + ))} +
+
+
+ ); +} + +function DangerZoneSkeleton() { + return ( + + + + + + + + + + + + + + ); +} + +export default function Loading() { + return ( + <> + + +
+ Loading server details +
+ + ); +} diff --git a/web/app/(dashboard)/layout-client.tsx b/web/app/(dashboard)/layout-client.tsx index a081f5da..12aa1b62 100644 --- a/web/app/(dashboard)/layout-client.tsx +++ b/web/app/(dashboard)/layout-client.tsx @@ -9,6 +9,7 @@ import { BreadcrumbDataProvider, useBreadcrumbs, } from "@/components/core/breadcrumb-data"; +import { DashboardPageSkeleton } from "@/components/dashboard/dashboard-page-skeleton"; import { OfflineServersBanner } from "@/components/server/offline-servers-banner"; import { Button } from "@/components/ui/button"; import { @@ -26,6 +27,10 @@ import { signOut, useSession } from "@/lib/auth-client"; function DashboardHeader({ email }: { email: string }) { const router = useRouter(); const breadcrumbs = useBreadcrumbs(); + const getBreadcrumbKey = ( + crumb: (typeof breadcrumbs)[number], + index: number, + ) => `${crumb.href}:${crumb.label}:${index}`; const mobileBreadcrumbs = breadcrumbs.length > 2 ? breadcrumbs.slice(-2) : breadcrumbs; @@ -48,7 +53,10 @@ function DashboardHeader({ email }: { email: string }) { <>