diff --git a/.agents/resume b/.agents/resume new file mode 100755 index 00000000..f29d5fe2 --- /dev/null +++ b/.agents/resume @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "No persistent services require repair." diff --git a/.agents/setup b/.agents/setup new file mode 100755 index 00000000..81a7777d --- /dev/null +++ b/.agents/setup @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +mise_bin="$HOME/.local/bin/mise" + +echo "Installing mise..." +if [[ ! -x "$mise_bin" ]]; then + curl -fsSL https://mise.run | sh +fi + +profile_marker="# Techulus Cloud toolchains managed by mise" +if ! grep -Fqx "$profile_marker" "$HOME/.bash_profile" 2>/dev/null; then + cat >> "$HOME/.bash_profile" <<'EOF' + +# Techulus Cloud toolchains managed by mise +if [[ -x "$HOME/.local/bin/mise" ]]; then + eval "$("$HOME/.local/bin/mise" activate bash)" +fi +EOF +fi + +go_version="$(awk '$1 == "go" { print $2; exit }' "$repo_root/agent/go.mod")" + +echo "Installing repository toolchains..." +"$mise_bin" use --global "go@$go_version" node@24 pnpm@11 +for config in \ + "$repo_root/agent/mise.toml" \ + "$repo_root/web/mise.toml" \ + "$repo_root/docs/mise.toml"; do + "$mise_bin" trust "$config" + ( + cd "$(dirname "$config")" + "$mise_bin" install + ) +done + +echo "Installing web dependencies..." +( + cd "$repo_root/web" + "$mise_bin" exec -- pnpm install --frozen-lockfile +) + +echo "Downloading Go dependencies..." +for module in agent cli deployment/updater; do + ( + cd "$repo_root/$module" + "$mise_bin" exec -- go mod download + ) +done + +echo "Orb setup complete." diff --git a/AGENT.md b/AGENT.md new file mode 100644 index 00000000..001a0b94 --- /dev/null +++ b/AGENT.md @@ -0,0 +1,43 @@ +# Techulus Cloud — Agent Guide + +An open container deployment platform. See README.md for architecture. + +## Project facts + +- **Unreleased beta.** Never build + backward-compatibility shims, deprecation windows, or migration paths for + old agent or API versions — delete and replace outright. +- The control plane and agent ship together; cross-cutting protocol changes + land in one PR with no rollout ordering concerns. + +## Repo map + +- `web/` — Next.js control plane (PostgreSQL + Drizzle, Inngest workflows) +- `agent/` — Go server agent (Podman, Traefik, WireGuard) +- `cli/` — Go CLI +- `deployment/` — production Compose files and updater +- `proxy/`, `registry/`, `logging/` — supporting service configs +- `docs/` — documentation + +## Commands + +- Web tests: `cd web && pnpm test` +- Web typecheck: `cd web && ./node_modules/.bin/tsc --noEmit` +- Web lint/format: `cd web && npx biome check --write ` +- Go (agent/cli): `go build ./...`, `go test ./...`, `gofmt -l .` +- After deleting or renaming a Next.js route, stale generated types in + `web/.next/types` can fail the typecheck — delete them; they regenerate. + +## Making changes + +- Pull latest main before starting; if there are conflicts, STOP. +- If product or architectural intent is unclear, ask — don't guess. +- Create a branch before committing; never commit to main or a release branch. +- Tests are expensive to write and maintain. Only add or expand tests for + high-value critical behavior, serious regression risk, or contracts that + would be costly to break. Keep tests focused; avoid low-signal harnesses. + +## ⚠️ Critical restrictions + +- **NEVER run the Node application** (`next dev`, `next start`, `pnpm dev`), Go Agent or Go CLI + without explicit permission. Tests, typechecks, and `go build` are fine. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..ac534a31 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENT.md \ No newline at end of file diff --git a/agent/README.md b/agent/README.md index fc1386e6..a193d024 100644 --- a/agent/README.md +++ b/agent/README.md @@ -44,13 +44,9 @@ curl -sSL $CONTROL_PLANE_URL/setup.sh | sudo bash ## Updating -To update an existing agent to the latest version: +When an update is available, open the server in the control plane and click **Update**. After you confirm the target version, the control plane queues an upgrade work item over the authenticated agent channel. -```bash -curl -sSL https://your-control-plane.com/update.sh | sudo bash -``` - -This downloads the latest agent binary, verifies the checksum, and restarts the service. +The agent downloads the release binary, verifies its checksum, installs it, and restarts itself. ## Manual Setup diff --git a/agent/internal/agent/agent.go b/agent/internal/agent/agent.go index 1c35e9bc..5a56fa73 100644 --- a/agent/internal/agent/agent.go +++ b/agent/internal/agent/agent.go @@ -72,6 +72,9 @@ type Agent struct { pendingServerlessWake map[string]serverlessTransitionGuard expectedStateMutex sync.RWMutex latestExpectedState *agenthttp.ExpectedState + compiledTraefikMutex sync.Mutex + compiledTraefikFor *agenthttp.ExpectedState + compiledTraefik *compiledTraefikState Client *agenthttp.Client Reconciler *reconcile.Reconciler Config *Config diff --git a/agent/internal/agent/drift.go b/agent/internal/agent/drift.go index c0c58341..fe5a3b70 100644 --- a/agent/internal/agent/drift.go +++ b/agent/internal/agent/drift.go @@ -353,28 +353,19 @@ func (a *Agent) planReconcile(expected *agenthttp.ExpectedState, actual *ActualS } if a.IsProxy { - expectedHttpRoutes := ConvertToHttpRoutes(expected.Traefik.HttpRoutes) - expectedTraefikHash := traefik.HashRoutesWithServerName(expectedHttpRoutes, expected.ServerName) - tcpRoutes := ConvertToTCPRoutes(expected.Traefik.TCPRoutes) - udpRoutes := ConvertToUDPRoutes(expected.Traefik.UDPRoutes) - expectedL4Hash := traefik.HashTCPRoutes(tcpRoutes) + traefik.HashUDPRoutes(udpRoutes) - expectedCerts := make([]traefik.Certificate, len(expected.Traefik.Certificates)) - for i, c := range expected.Traefik.Certificates { - expectedCerts[i] = traefik.Certificate{Domain: c.Domain, Certificate: c.Certificate, CertificateKey: c.CertificateKey} - } - expectedCertsHash := traefik.HashCertificates(expectedCerts) - if expectedTraefikHash != actual.TraefikConfigHash || - expectedL4Hash != actual.L4ConfigHash || - expectedCertsHash != actual.CertificatesHash || + compiled := a.compiledTraefikState(expected) + if compiled.HTTPHash != actual.TraefikConfigHash || + compiled.L4Hash != actual.L4ConfigHash || + compiled.CertHash != actual.CertificatesHash || !actual.TraefikReloaded { actions = append(actions, reconcileAction{ Kind: actionUpdateTraefik, Description: fmt.Sprintf( "UPDATE Traefik (%d HTTP, %d TCP, %d UDP routes; %d certificates)", - len(expected.Traefik.HttpRoutes), - len(tcpRoutes), - len(udpRoutes), - len(expected.Traefik.Certificates), + len(compiled.HTTP), + len(compiled.TCP), + len(compiled.UDP), + len(compiled.Certificates), ), }) } @@ -570,17 +561,7 @@ func (a *Agent) applyReconcileAction(action reconcileAction) error { } func (a *Agent) updateTraefik() error { - expectedHttpRoutes := ConvertToHttpRoutes(a.expectedState.Traefik.HttpRoutes) - tcpRoutes := ConvertToTCPRoutes(a.expectedState.Traefik.TCPRoutes) - udpRoutes := ConvertToUDPRoutes(a.expectedState.Traefik.UDPRoutes) - - var tcpPorts, udpPorts []int - for _, r := range tcpRoutes { - tcpPorts = append(tcpPorts, r.ExternalPort) - } - for _, r := range udpRoutes { - udpPorts = append(udpPorts, r.ExternalPort) - } + compiled := a.compiledTraefikState(a.expectedState) needsRestart := false metricsRestart, err := traefik.EnsureMetricsConfig() @@ -589,9 +570,9 @@ func (a *Agent) updateTraefik() error { } needsRestart = metricsRestart - if len(tcpPorts) > 0 || len(udpPorts) > 0 { - log.Printf("[reconcile] ensuring L4 entry points: %d TCP, %d UDP", len(tcpPorts), len(udpPorts)) - entryPointsRestart, err := traefik.EnsureEntryPoints(tcpPorts, udpPorts) + if len(compiled.TCPPorts) > 0 || len(compiled.UDPPorts) > 0 { + log.Printf("[reconcile] ensuring L4 entry points: %d TCP, %d UDP", len(compiled.TCPPorts), len(compiled.UDPPorts)) + entryPointsRestart, err := traefik.EnsureEntryPoints(compiled.TCPPorts, compiled.UDPPorts) if err != nil { return fmt.Errorf("failed to ensure entry points: %w", err) } @@ -604,19 +585,9 @@ func (a *Agent) updateTraefik() error { } } - expectedCerts := make([]traefik.Certificate, len(a.expectedState.Traefik.Certificates)) - for i, certificate := range a.expectedState.Traefik.Certificates { - expectedCerts[i] = traefik.Certificate{ - Domain: certificate.Domain, - Certificate: certificate.Certificate, - CertificateKey: certificate.CertificateKey, - } - } - expectedTraefikHash := traefik.HashRoutesWithServerName(expectedHttpRoutes, a.expectedState.ServerName) - expectedL4Hash := traefik.HashTCPRoutes(tcpRoutes) + traefik.HashUDPRoutes(udpRoutes) - routesChanged := expectedTraefikHash != traefik.GetCurrentConfigHash() || - expectedL4Hash != traefik.GetCurrentL4ConfigHash() - certificatesChanged := traefik.HashCertificates(expectedCerts) != traefik.GetCurrentCertificatesHash() + routesChanged := compiled.HTTPHash != traefik.GetCurrentConfigHash() || + compiled.L4Hash != traefik.GetCurrentL4ConfigHash() + certificatesChanged := compiled.CertHash != traefik.GetCurrentCertificatesHash() if !routesChanged && !certificatesChanged { if err := traefik.EnsureDynamicConfigReloaded(a.DataDir, 15*time.Second); err != nil { return fmt.Errorf("failed to recover Traefik config reload: %w", err) @@ -633,13 +604,13 @@ func (a *Agent) updateTraefik() error { } if certificatesChanged { - if err := traefik.UpdateCertificates(expectedCerts); err != nil { + if err := traefik.UpdateCertificates(compiled.Certificates); err != nil { return fmt.Errorf("failed to update Traefik certificates: %w", err) } } if routesChanged { - log.Printf("[reconcile] updating Traefik routes (HTTP: %d, TCP: %d, UDP: %d)", len(expectedHttpRoutes), len(tcpRoutes), len(udpRoutes)) - if err := traefik.UpdateHttpRoutesWithL4(expectedHttpRoutes, tcpRoutes, udpRoutes, a.expectedState.ServerName); err != nil { + log.Printf("[reconcile] updating Traefik routes (HTTP: %d, TCP: %d, UDP: %d)", len(compiled.HTTP), len(compiled.TCP), len(compiled.UDP)) + if err := traefik.UpdateHttpRoutesWithL4(compiled.HTTP, compiled.TCP, compiled.UDP, a.expectedState.ServerName); err != nil { return fmt.Errorf("failed to update Traefik: %w", err) } } diff --git a/agent/internal/agent/helpers.go b/agent/internal/agent/helpers.go index 29a0131d..fecae650 100644 --- a/agent/internal/agent/helpers.go +++ b/agent/internal/agent/helpers.go @@ -50,3 +50,62 @@ func ConvertToUDPRoutes(routes []agenthttp.TraefikUDPRoute) []traefik.TraefikUDP } return udpRoutes } + +type compiledTraefikState struct { + HTTP []traefik.TraefikRoute + TCP []traefik.TraefikTCPRoute + UDP []traefik.TraefikUDPRoute + Certificates []traefik.Certificate + TCPPorts []int + UDPPorts []int + + HTTPHash string + L4Hash string + CertHash string +} + +func compileTraefikState(expected *agenthttp.ExpectedState) *compiledTraefikState { + httpRoutes := ConvertToHttpRoutes(expected.Traefik.HttpRoutes) + tcpRoutes := ConvertToTCPRoutes(expected.Traefik.TCPRoutes) + udpRoutes := ConvertToUDPRoutes(expected.Traefik.UDPRoutes) + + certificates := make([]traefik.Certificate, len(expected.Traefik.Certificates)) + for i, c := range expected.Traefik.Certificates { + certificates[i] = traefik.Certificate{ + Domain: c.Domain, + Certificate: c.Certificate, + CertificateKey: c.CertificateKey, + } + } + + var tcpPorts, udpPorts []int + for _, r := range tcpRoutes { + tcpPorts = append(tcpPorts, r.ExternalPort) + } + for _, r := range udpRoutes { + udpPorts = append(udpPorts, r.ExternalPort) + } + + return &compiledTraefikState{ + HTTP: httpRoutes, + TCP: tcpRoutes, + UDP: udpRoutes, + Certificates: certificates, + TCPPorts: tcpPorts, + UDPPorts: udpPorts, + HTTPHash: traefik.HashRoutesWithServerName(httpRoutes, expected.ServerName), + L4Hash: traefik.HashTCPRoutes(tcpRoutes) + traefik.HashUDPRoutes(udpRoutes), + CertHash: traefik.HashCertificates(certificates), + } +} + +func (a *Agent) compiledTraefikState(expected *agenthttp.ExpectedState) *compiledTraefikState { + a.compiledTraefikMutex.Lock() + defer a.compiledTraefikMutex.Unlock() + + if a.compiledTraefikFor != expected || a.compiledTraefik == nil { + a.compiledTraefik = compileTraefikState(expected) + a.compiledTraefikFor = expected + } + return a.compiledTraefik +} diff --git a/agent/internal/agent/reporting.go b/agent/internal/agent/reporting.go index 9a3b5cb3..ccd45efa 100644 --- a/agent/internal/agent/reporting.go +++ b/agent/internal/agent/reporting.go @@ -89,11 +89,26 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport if a.ShouldSuppressServerlessContainerReport(c.DeploymentID) { continue } - - status := "stopped" - if c.State == "running" { + // Intermediate podman states (e.g. "created" mid-deploy) must not be + // reported as stopped — the control plane would move the deployment + // into a stopped phase — nor omitted, which would read as the + // container being gone. They are reported as "transient" so the + // control plane keeps tracking the deployment without acting until + // the state settles. Settled non-running states ("stopped", + // "paused") map to stopped so the deployment leaves routing and + // drift reconciliation can repair it; the same goes for "unknown" + // or unrecognized states, since presence-only reporting there would + // leave a broken container marked healthy indefinitely. + var status string + switch c.State { + case "running": status = "running" - } else if c.State == "exited" { + case "exited", "stopped", "paused": + status = "stopped" + case "created", "configured", "initialized", "stopping", "removing": + status = "transient" + default: + log.Printf("[status] container %s in unexpected state %q, reporting as stopped", c.ID, c.State) status = "stopped" } @@ -148,24 +163,14 @@ func (a *Agent) routingSyncedRolloutIds() []string { } func (a *Agent) proxyRoutingStateConverged(expected *agenthttp.ExpectedState) bool { - httpRoutes := ConvertToHttpRoutes(expected.Traefik.HttpRoutes) - if traefik.HashRoutesWithServerName(httpRoutes, expected.ServerName) != traefik.GetCurrentConfigHash() { + compiled := a.compiledTraefikState(expected) + if compiled.HTTPHash != traefik.GetCurrentConfigHash() { return false } - tcpRoutes := ConvertToTCPRoutes(expected.Traefik.TCPRoutes) - udpRoutes := ConvertToUDPRoutes(expected.Traefik.UDPRoutes) - if traefik.HashTCPRoutes(tcpRoutes)+traefik.HashUDPRoutes(udpRoutes) != traefik.GetCurrentL4ConfigHash() { + if compiled.L4Hash != traefik.GetCurrentL4ConfigHash() { return false } - certificates := make([]traefik.Certificate, len(expected.Traefik.Certificates)) - for i, certificate := range expected.Traefik.Certificates { - certificates[i] = traefik.Certificate{ - Domain: certificate.Domain, - Certificate: certificate.Certificate, - CertificateKey: certificate.CertificateKey, - } - } - if traefik.HashCertificates(certificates) != traefik.GetCurrentCertificatesHash() { + if compiled.CertHash != traefik.GetCurrentCertificatesHash() { return false } reloaded, err := traefik.DynamicConfigReloaded(a.DataDir) diff --git a/agent/internal/agent/serverless.go b/agent/internal/agent/serverless.go index 7bbbc702..374d6b86 100644 --- a/agent/internal/agent/serverless.go +++ b/agent/internal/agent/serverless.go @@ -160,22 +160,18 @@ func (a *Agent) SnapshotServerlessTransitions() []agenthttp.ServerlessTransition return append([]agenthttp.ServerlessTransition(nil), a.pendingServerlessTransitions...) } -func (a *Agent) ClearReportedServerlessTransitions(count int) { - if count <= 0 { - return - } - - a.serverlessMutex.Lock() - defer a.serverlessMutex.Unlock() - if count > len(a.pendingServerlessTransitions) { - count = len(a.pendingServerlessTransitions) - } - a.pendingServerlessTransitions = a.pendingServerlessTransitions[count:] -} - func (a *Agent) AcknowledgeServerlessTransitions(results []agenthttp.ServerlessTransitionResult, reportedCount int) { if len(results) == 0 { - a.ClearReportedServerlessTransitions(reportedCount) + if reportedCount <= 0 { + return + } + + a.serverlessMutex.Lock() + defer a.serverlessMutex.Unlock() + if reportedCount > len(a.pendingServerlessTransitions) { + reportedCount = len(a.pendingServerlessTransitions) + } + a.pendingServerlessTransitions = a.pendingServerlessTransitions[reportedCount:] return } diff --git a/agent/internal/container/runtime.go b/agent/internal/container/runtime.go index a0993d9a..297b97c9 100644 --- a/agent/internal/container/runtime.go +++ b/agent/internal/container/runtime.go @@ -303,41 +303,14 @@ func ForceRemove(containerID string) error { } func Restart(containerID string) error { - exists, err := ContainerExists(containerID) - if err != nil { - return fmt.Errorf("failed to check container existence: %w", err) - } - if !exists { - return fmt.Errorf("container does not exist: %s", containerID) - } - - log.Printf("[podman:restart] restarting container %s", containerID) - restartCmd := exec.Command("podman", "restart", containerID) - if output, err := restartCmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to restart container: %s: %w", string(output), err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - log.Printf("[podman:restart] verifying container %s is running", containerID) - err = retry.WithBackoff(ctx, retry.DeployBackoff, func() (bool, error) { - running, err := IsContainerRunning(containerID) - if err != nil { - return false, err - } - return running, nil - }) - - if err != nil { - return fmt.Errorf("container failed to restart: %w", err) - } - - log.Printf("[podman:restart] container %s restarted successfully", containerID) - return nil + return startOrRestart(containerID, "restart") } func Start(containerID string) error { + return startOrRestart(containerID, "start") +} + +func startOrRestart(containerID, operation string) error { exists, err := ContainerExists(containerID) if err != nil { return fmt.Errorf("failed to check container existence: %w", err) @@ -346,16 +319,16 @@ func Start(containerID string) error { return fmt.Errorf("container does not exist: %s", containerID) } - log.Printf("[podman:start] starting container %s", containerID) - startCmd := exec.Command("podman", "start", containerID) - if output, err := startCmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to start container: %s: %w", string(output), err) + log.Printf("[podman:%s] %sing container %s", operation, operation, containerID) + cmd := exec.Command("podman", operation, containerID) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to %s container: %s: %w", operation, string(output), err) } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - log.Printf("[podman:start] verifying container %s is running", containerID) + log.Printf("[podman:%s] verifying container %s is running", operation, containerID) err = retry.WithBackoff(ctx, retry.DeployBackoff, func() (bool, error) { running, err := IsContainerRunning(containerID) if err != nil { @@ -365,10 +338,10 @@ func Start(containerID string) error { }) if err != nil { - return fmt.Errorf("container failed to start: %w", err) + return fmt.Errorf("container failed to %s: %w", operation, err) } - log.Printf("[podman:start] container %s started successfully", containerID) + log.Printf("[podman:%s] container %s %sed successfully", operation, containerID, operation) return nil } diff --git a/agent/internal/container/stats_test.go b/agent/internal/container/stats_test.go index 1edfd61b..0f9844f5 100644 --- a/agent/internal/container/stats_test.go +++ b/agent/internal/container/stats_test.go @@ -1,13 +1,14 @@ package container -import "testing" +import ( + "reflect" + "testing" +) func TestParsePodmanStatsOutputArray(t *testing.T) { containers := []Container{ { ID: "abcdef1234567890", - Name: "api", - State: "running", ServiceID: "svc_1", DeploymentID: "dep_1", }, @@ -26,37 +27,24 @@ func TestParsePodmanStatsOutputArray(t *testing.T) { if err != nil { t.Fatalf("parse stats: %v", err) } - if len(stats) != 1 { - t.Fatalf("expected 1 stat, got %d", len(stats)) - } - - stat := stats[0] - if stat.ContainerID != "abcdef1234567890" { - t.Fatalf("container id = %q", stat.ContainerID) - } - if stat.ServiceID != "svc_1" || stat.DeploymentID != "dep_1" { - t.Fatalf("unexpected service/deployment labels: %#v", stat) - } - if stat.CPUUsagePercent != 12.34 { - t.Fatalf("cpu = %f", stat.CPUUsagePercent) - } - if stat.MemoryUsagePercent != 12.5 { - t.Fatalf("memory percent = %f", stat.MemoryUsagePercent) - } - if stat.MemoryUsedBytes != 64*1024*1024 { - t.Fatalf("memory bytes = %f", stat.MemoryUsedBytes) - } - if stat.NetworkReceiveBytes != 1.5*1000*1000 { - t.Fatalf("rx bytes = %f", stat.NetworkReceiveBytes) - } - if stat.NetworkTransmitBytes != 2.5*1000*1000 { - t.Fatalf("tx bytes = %f", stat.NetworkTransmitBytes) + want := []ResourceStats{{ + ContainerID: "abcdef1234567890", + ServiceID: "svc_1", + DeploymentID: "dep_1", + CPUUsagePercent: 12.34, + MemoryUsagePercent: 12.5, + MemoryUsedBytes: 64 * 1024 * 1024, + NetworkReceiveBytes: 1.5 * 1000 * 1000, + NetworkTransmitBytes: 2.5 * 1000 * 1000, + }} + if !reflect.DeepEqual(stats, want) { + t.Fatalf("stats = %#v, want %#v", stats, want) } } func TestParsePodmanStatsOutputJSONLines(t *testing.T) { containers := []Container{ - {ID: "1234567890abcdef", Name: "worker", State: "running", ServiceID: "svc_2", DeploymentID: "dep_2"}, + {ID: "1234567890abcdef", ServiceID: "svc_2", DeploymentID: "dep_2"}, } stats, err := parsePodmanStatsOutput([]byte(`{"ContainerID":"1234567890","CPUPerc":"0%","MemUsage":"128MB / 1GB","MemPerc":"10%","NetIO":"0B / 32kB"}`), containers) @@ -76,8 +64,8 @@ func TestParsePodmanStatsOutputJSONLines(t *testing.T) { func TestParsePodmanStatsOutputNameFallbackDoesNotUseIDPrefix(t *testing.T) { containers := []Container{ - {ID: "api1234567890", Name: "backend", State: "running", ServiceID: "wrong", DeploymentID: "wrong_dep"}, - {ID: "fedcba987654", Name: "api", State: "running", ServiceID: "svc_3", DeploymentID: "dep_3"}, + {ID: "api1234567890", Name: "backend", ServiceID: "wrong", DeploymentID: "wrong_dep"}, + {ID: "fedcba987654", Name: "api", ServiceID: "svc_3", DeploymentID: "dep_3"}, } stats, err := parsePodmanStatsOutput([]byte(`[ diff --git a/agent/internal/dns/store.go b/agent/internal/dns/store.go index fb0f01b2..b4c54d3a 100644 --- a/agent/internal/dns/store.go +++ b/agent/internal/dns/store.go @@ -1,10 +1,7 @@ package dns import ( - "crypto/sha256" - "encoding/hex" "net" - "sort" "strings" "sync" "sync/atomic" @@ -48,7 +45,7 @@ func (s *RecordStore) Update(records []DnsRecord) { s.records = newRecords s.rrIndex = newRRIndex - s.hash = hashRecordsInternal(records) + s.hash = HashRecords(records) } func (s *RecordStore) Lookup(name string) []net.IP { @@ -87,24 +84,3 @@ func normalizeName(name string) string { } return name } - -func hashRecordsInternal(records []DnsRecord) string { - sortedRecords := make([]DnsRecord, len(records)) - copy(sortedRecords, records) - sort.Slice(sortedRecords, func(i, j int) bool { - return sortedRecords[i].Name < sortedRecords[j].Name - }) - - var sb strings.Builder - for _, r := range sortedRecords { - sb.WriteString(r.Name) - sb.WriteString(":") - sortedIps := make([]string, len(r.Ips)) - copy(sortedIps, r.Ips) - sort.Strings(sortedIps) - sb.WriteString(strings.Join(sortedIps, ",")) - sb.WriteString("|") - } - hash := sha256.Sum256([]byte(sb.String())) - return hex.EncodeToString(hash[:]) -} diff --git a/agent/internal/dns/store_test.go b/agent/internal/dns/store_test.go new file mode 100644 index 00000000..dc6cb6ca --- /dev/null +++ b/agent/internal/dns/store_test.go @@ -0,0 +1,27 @@ +package dns + +import ( + "reflect" + "testing" +) + +func TestRecordStoreUpdateHashMatchesHashRecordsWithoutMutatingInput(t *testing.T) { + records := []DnsRecord{ + {Name: "z.internal", Ips: []string{"10.0.0.2", "10.0.0.1"}}, + {Name: "a.internal", Ips: []string{"10.0.1.2", "10.0.1.1"}}, + } + original := []DnsRecord{ + {Name: records[0].Name, Ips: append([]string(nil), records[0].Ips...)}, + {Name: records[1].Name, Ips: append([]string(nil), records[1].Ips...)}, + } + + store := NewRecordStore() + store.Update(records) + + if got, want := store.Hash(), HashRecords(records); got != want { + t.Fatalf("hash = %q, want %q", got, want) + } + if !reflect.DeepEqual(records, original) { + t.Fatalf("Update mutated records: got %#v, want %#v", records, original) + } +} diff --git a/agent/internal/http/client.go b/agent/internal/http/client.go index 47dbbcbe..802e2d4c 100644 --- a/agent/internal/http/client.go +++ b/agent/internal/http/client.go @@ -17,12 +17,11 @@ import ( ) type Client struct { - baseURL string - serverID string - keyPair *crypto.KeyPair - client *http.Client - longClient *http.Client - dataDir string + baseURL string + serverID string + keyPair *crypto.KeyPair + client *http.Client + dataDir string } func NewClient(baseURL, serverID string, keyPair *crypto.KeyPair, dataDir string) *Client { @@ -34,9 +33,6 @@ func NewClient(baseURL, serverID string, keyPair *crypto.KeyPair, dataDir string client: &http.Client{ Timeout: 30 * time.Second, }, - longClient: &http.Client{ - Timeout: 40 * time.Second, - }, } } @@ -50,6 +46,27 @@ func (c *Client) signRequest(req *http.Request, body string) { req.Header.Set("x-signature", signature) } +func (c *Client) doSignedJSONRequest(url string, body []byte, acceptedStatuses []int, requestError, responseError string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + c.signRequest(req, string(body)) + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("%s: %w", requestError, err) + } + for _, status := range acceptedStatuses { + if resp.StatusCode == status { + return resp, nil + } + } + defer resp.Body.Close() + respBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("%s with status %d: %s", responseError, resp.StatusCode, string(respBody)) +} + type PortMapping struct { ContainerPort int `json:"containerPort"` HostPort int `json:"hostPort"` @@ -368,25 +385,12 @@ func (c *Client) UpdateBuildStatus(buildID, status, errorMsg, resolvedCommitSha return fmt.Errorf("failed to marshal status update: %w", err) } - req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/builds/"+buildID+"/status", bytes.NewReader(body)) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - c.signRequest(req, string(body)) - - resp, err := c.client.Do(req) + resp, err := c.doSignedJSONRequest(c.baseURL+"/api/v1/agent/builds/"+buildID+"/status", body, []int{http.StatusOK}, "failed to update build status", "build status update failed") if err != nil { - return fmt.Errorf("failed to update build status: %w", err) + return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - respBody, _ := io.ReadAll(resp.Body) - return fmt.Errorf("build status update failed with status %d: %s", resp.StatusCode, string(respBody)) - } - return nil } @@ -438,25 +442,12 @@ func (c *Client) ReportStatus(report *StatusReport, completed []CompletedWorkIte return nil, fmt.Errorf("failed to marshal status report: %w", err) } - req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/status", bytes.NewReader(body)) + resp, err := c.doSignedJSONRequest(c.baseURL+"/api/v1/agent/status", body, []int{http.StatusOK, http.StatusAccepted}, "failed to report status", "status report failed") if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - c.signRequest(req, string(body)) - - resp, err := c.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to report status: %w", err) + return nil, err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { - respBody, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("status report failed with status %d: %s", resp.StatusCode, string(respBody)) - } - var statusResponse StatusResponse if err := json.NewDecoder(resp.Body).Decode(&statusResponse); err != nil { return nil, fmt.Errorf("failed to decode status response: %w", err) @@ -506,25 +497,12 @@ func (c *Client) ReportBackupComplete(backupID string, sizeBytes int64, checksum return fmt.Errorf("failed to marshal backup complete: %w", err) } - req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/backup/complete", bytes.NewReader(body)) + resp, err := c.doSignedJSONRequest(c.baseURL+"/api/v1/agent/backup/complete", body, []int{http.StatusOK}, "failed to report backup complete", "backup complete report failed") if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - c.signRequest(req, string(body)) - - resp, err := c.client.Do(req) - if err != nil { - return fmt.Errorf("failed to report backup complete: %w", err) + return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - respBody, _ := io.ReadAll(resp.Body) - return fmt.Errorf("backup complete report failed with status %d: %s", resp.StatusCode, string(respBody)) - } - return nil } @@ -539,25 +517,12 @@ func (c *Client) ReportBackupFailed(backupID string, errorMsg string) error { return fmt.Errorf("failed to marshal backup failed: %w", err) } - req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/backup/failed", bytes.NewReader(body)) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - c.signRequest(req, string(body)) - - resp, err := c.client.Do(req) + resp, err := c.doSignedJSONRequest(c.baseURL+"/api/v1/agent/backup/failed", body, []int{http.StatusOK}, "failed to report backup failed", "backup failed report failed") if err != nil { - return fmt.Errorf("failed to report backup failed: %w", err) + return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - respBody, _ := io.ReadAll(resp.Body) - return fmt.Errorf("backup failed report failed with status %d: %s", resp.StatusCode, string(respBody)) - } - return nil } @@ -575,24 +540,11 @@ func (c *Client) ReportRestoreComplete(backupID string, success bool, errorMsg s return fmt.Errorf("failed to marshal restore complete: %w", err) } - req, err := http.NewRequest("POST", c.baseURL+"/api/v1/agent/restore/complete", bytes.NewReader(body)) + resp, err := c.doSignedJSONRequest(c.baseURL+"/api/v1/agent/restore/complete", body, []int{http.StatusOK}, "failed to report restore complete", "restore complete report failed") if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - c.signRequest(req, string(body)) - - resp, err := c.client.Do(req) - if err != nil { - return fmt.Errorf("failed to report restore complete: %w", err) + return err } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - respBody, _ := io.ReadAll(resp.Body) - return fmt.Errorf("restore complete report failed with status %d: %s", resp.StatusCode, string(respBody)) - } - return nil } diff --git a/agent/internal/http/client_test.go b/agent/internal/http/client_test.go new file mode 100644 index 00000000..3ad5c145 --- /dev/null +++ b/agent/internal/http/client_test.go @@ -0,0 +1,40 @@ +package http + +import ( + "io" + stdhttp "net/http" + "net/http/httptest" + "testing" + + "techulus/cloud-agent/internal/crypto" +) + +func TestSignedJSONRequests(t *testing.T) { + keyPair, err := crypto.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(stdhttp.HandlerFunc(func(w stdhttp.ResponseWriter, r *stdhttp.Request) { + body, _ := io.ReadAll(r.Body) + if r.URL.Path == "/api/v1/agent/status" { + message := r.Header.Get("x-timestamp") + ":" + string(body) + if r.Method != stdhttp.MethodPost || r.Header.Get("Content-Type") != "application/json" || r.Header.Get("x-server-id") != "server-1" || r.Header.Get("x-signature") != keyPair.Sign([]byte(message)) { + t.Error("request method, headers, or signature not preserved") + } + w.WriteHeader(stdhttp.StatusAccepted) + io.WriteString(w, "{\"ok\":true}") + return + } + w.WriteHeader(stdhttp.StatusTeapot) + io.WriteString(w, "backup rejected") + })) + defer server.Close() + client := NewClient(server.URL, "server-1", keyPair, "") + response, err := client.ReportStatus(&StatusReport{}, nil, nil, nil) + if err != nil || !response.OK { + t.Fatalf("accepted response was not decoded: response=%+v err=%v", response, err) + } + if err := client.ReportBackupFailed("backup-1", "failed"); err == nil || err.Error() != "backup failed report failed with status 418: backup rejected" { + t.Fatalf("unexpected error response: %v", err) + } +} diff --git a/agent/internal/logs/traefik_collector.go b/agent/internal/logs/traefik_collector.go index 7a3dc5aa..182f4622 100644 --- a/agent/internal/logs/traefik_collector.go +++ b/agent/internal/logs/traefik_collector.go @@ -37,38 +37,16 @@ type HTTPLogSender interface { } type TraefikLogEntry struct { - ClientAddr string `json:"ClientAddr"` - ClientHost string `json:"ClientHost"` - ClientPort string `json:"ClientPort"` - ClientUsername string `json:"ClientUsername"` - DownstreamContentSize int `json:"DownstreamContentSize"` - DownstreamStatus int `json:"DownstreamStatus"` - Duration int64 `json:"Duration"` - OriginContentSize int `json:"OriginContentSize"` - OriginDuration int64 `json:"OriginDuration"` - OriginStatus int `json:"OriginStatus"` - Overhead int64 `json:"Overhead"` - RequestAddr string `json:"RequestAddr"` - RequestContentSize int `json:"RequestContentSize"` - RequestCount int `json:"RequestCount"` - RequestHost string `json:"RequestHost"` - RequestMethod string `json:"RequestMethod"` - RequestPath string `json:"RequestPath"` - RequestPort string `json:"RequestPort"` - RequestProtocol string `json:"RequestProtocol"` - RequestScheme string `json:"RequestScheme"` - RetryAttempts int `json:"RetryAttempts"` - RouterName string `json:"RouterName"` - ServiceAddr string `json:"ServiceAddr"` - ServiceName string `json:"ServiceName"` - ServiceURL string `json:"ServiceURL"` - StartLocal string `json:"StartLocal"` - StartUTC string `json:"StartUTC"` - TLSCipher string `json:"TLSCipher"` - TLSVersion string `json:"TLSVersion"` - Time string `json:"time"` - Level string `json:"level"` - Msg string `json:"msg"` + ClientHost string `json:"ClientHost"` + DownstreamContentSize int `json:"DownstreamContentSize"` + DownstreamStatus int `json:"DownstreamStatus"` + Duration int64 `json:"Duration"` + RequestHost string `json:"RequestHost"` + RequestMethod string `json:"RequestMethod"` + RequestPath string `json:"RequestPath"` + RouterName string `json:"RouterName"` + StartUTC string `json:"StartUTC"` + Time string `json:"time"` } type TraefikCollector struct { diff --git a/agent/internal/logs/traefik_collector_test.go b/agent/internal/logs/traefik_collector_test.go new file mode 100644 index 00000000..45f7ee1a --- /dev/null +++ b/agent/internal/logs/traefik_collector_test.go @@ -0,0 +1,41 @@ +package logs + +import ( + "reflect" + "testing" +) + +func TestTraefikCollectorProcessLineQueuesRetainedFields(t *testing.T) { + collector := NewTraefikCollector(nil) + collector.processLine([]byte(`{ + "ClientHost":"192.0.2.10", + "DownstreamContentSize":321, + "DownstreamStatus":201, + "Duration":12500000, + "RequestHost":"api.example.com", + "RequestMethod":"POST", + "RequestPath":"/v1/items", + "RouterName":"service-42@docker", + "StartUTC":"2026-07-23T10:11:12Z", + "time":"", + "ClientAddr":"ignored:1234", + "OriginStatus":503, + "TLSVersion":"1.3", + "msg":"ignored" + }`)) + + want := []HTTPLogEntry{{ + ServiceId: "service-42", + Host: "api.example.com", + Method: "POST", + Path: "/v1/items", + Status: 201, + Duration: 12.5, + Size: 321, + ClientIP: "192.0.2.10", + Timestamp: "2026-07-23T10:11:12Z", + }} + if !reflect.DeepEqual(collector.queue, want) { + t.Fatalf("queued entries = %#v, want %#v", collector.queue, want) + } +} diff --git a/agent/internal/logs/victoria.go b/agent/internal/logs/victoria.go index df845b67..8e123f1b 100644 --- a/agent/internal/logs/victoria.go +++ b/agent/internal/logs/victoria.go @@ -47,6 +47,32 @@ func (v *VictoriaLogsSender) setAuthHeader(req *http.Request) { } } +func (v *VictoriaLogsSender) post(body []byte, transportError string) (int, time.Duration, error) { + req, err := http.NewRequest("POST", v.endpoint+"/insert/jsonline", bytes.NewReader(body)) + if err != nil { + return 0, 0, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + v.setAuthHeader(req) + + start := time.Now() + resp, err := v.client.Do(req) + elapsed := time.Since(start) + if err != nil { + return 0, elapsed, fmt.Errorf("%s: %w", transportError, err) + } + defer resp.Body.Close() + + return resp.StatusCode, elapsed, nil +} + +func acceptedVictoriaStatus(status int) error { + if status != http.StatusOK && status != http.StatusNoContent { + return fmt.Errorf("unexpected status code: %d", status) + } + return nil +} + type victoriaLogEntry struct { Msg string `json:"_msg"` Time string `json:"_time"` @@ -89,27 +115,12 @@ func (v *VictoriaLogsSender) SendLogs(batch *LogBatch) error { url := v.endpoint + "/insert/jsonline" log.Printf("[logs] sending %d logs (%d bytes) to %s", len(batch.Logs), buf.Len(), url) - req, err := http.NewRequest("POST", url, &buf) + status, elapsed, err := v.post(buf.Bytes(), "failed to send logs") if err != nil { - return fmt.Errorf("failed to create request: %w", err) + return err } - req.Header.Set("Content-Type", "application/json") - v.setAuthHeader(req) - - start := time.Now() - resp, err := v.client.Do(req) - if err != nil { - return fmt.Errorf("failed to send logs: %w", err) - } - defer resp.Body.Close() - - log.Printf("[logs] response: %d in %v", resp.StatusCode, time.Since(start)) - - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - return nil + log.Printf("[logs] response: %d in %v", status, elapsed) + return acceptedVictoriaStatus(status) } type victoriaHTTPLogEntry struct { @@ -156,27 +167,12 @@ func (v *VictoriaLogsSender) SendHTTPLogs(logs []HTTPLogEntry) error { url := v.endpoint + "/insert/jsonline" log.Printf("[traefik-logs] sending %d HTTP logs (%d bytes) to %s", len(logs), buf.Len(), url) - req, err := http.NewRequest("POST", url, &buf) + status, elapsed, err := v.post(buf.Bytes(), "failed to send HTTP logs") if err != nil { - return fmt.Errorf("failed to create request: %w", err) + return err } - req.Header.Set("Content-Type", "application/json") - v.setAuthHeader(req) - - start := time.Now() - resp, err := v.client.Do(req) - if err != nil { - return fmt.Errorf("failed to send HTTP logs: %w", err) - } - defer resp.Body.Close() - - log.Printf("[traefik-logs] response: %d in %v", resp.StatusCode, time.Since(start)) - - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - return nil + log.Printf("[traefik-logs] response: %d in %v", status, elapsed) + return acceptedVictoriaStatus(status) } type victoriaBuildLogEntry struct { @@ -229,26 +225,11 @@ func (v *VictoriaLogsSender) SendBuildLogs(buildID, serviceID, projectID string, return nil } - url := v.endpoint + "/insert/jsonline" - - req, err := http.NewRequest("POST", url, &buf) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - v.setAuthHeader(req) - - resp, err := v.client.Do(req) + status, _, err := v.post(buf.Bytes(), "failed to send build logs") if err != nil { - return fmt.Errorf("failed to send build logs: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return err } - - return nil + return acceptedVictoriaStatus(status) } func (v *VictoriaLogsSender) SendAgentLogs(logs []AgentLog) error { @@ -273,24 +254,9 @@ func (v *VictoriaLogsSender) SendAgentLogs(logs []AgentLog) error { return nil } - url := v.endpoint + "/insert/jsonline" - - req, err := http.NewRequest("POST", url, &buf) + status, _, err := v.post(buf.Bytes(), "failed to send agent logs") if err != nil { - return fmt.Errorf("failed to create request: %w", err) + return err } - req.Header.Set("Content-Type", "application/json") - v.setAuthHeader(req) - - resp, err := v.client.Do(req) - if err != nil { - return fmt.Errorf("failed to send agent logs: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - return nil + return acceptedVictoriaStatus(status) } diff --git a/agent/internal/logs/victoria_test.go b/agent/internal/logs/victoria_test.go new file mode 100644 index 00000000..ba0ca202 --- /dev/null +++ b/agent/internal/logs/victoria_test.go @@ -0,0 +1,112 @@ +package logs + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestVictoriaSendMethodsPostJSONLines(t *testing.T) { + tests := []struct { + name string + send func(*VictoriaLogsSender) error + }{ + { + name: "container", + send: func(v *VictoriaLogsSender) error { + return v.SendLogs(&LogBatch{Logs: []LogEntry{{Message: "container ready"}}}) + }, + }, + { + name: "HTTP", + send: func(v *VictoriaLogsSender) error { + return v.SendHTTPLogs([]HTTPLogEntry{{Method: "GET", Path: "/health", Status: 200}}) + }, + }, + { + name: "build", + send: func(v *VictoriaLogsSender) error { + return v.SendBuildLogs("build-1", "service-1", "project-1", []string{"build ready"}) + }, + }, + { + name: "agent", + send: func(v *VictoriaLogsSender) error { + return v.SendAgentLogs([]AgentLog{{Message: "agent ready"}}) + }, + }, + } + + for _, status := range []int{http.StatusOK, http.StatusNoContent} { + for _, tt := range tests { + t.Run(tt.name+http.StatusText(status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + if r.URL.Path != "/insert/jsonline" { + t.Errorf("path = %q, want /insert/jsonline", r.URL.Path) + } + if got := r.Header.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q, want application/json", got) + } + username, password, ok := r.BasicAuth() + if !ok || username != "oracle" || password != "secret" { + t.Errorf("Basic Auth = %q, %q, %v", username, password, ok) + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if len(body) == 0 || body[0] != '{' || body[len(body)-1] != '\n' { + t.Errorf("body is not a non-empty JSON line: %q", body) + } + w.WriteHeader(status) + })) + defer server.Close() + + sender := NewVictoriaLogsSender(strings.Replace(server.URL, "://", "://oracle:secret@", 1), "server-1") + if err := tt.send(sender); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestSendHTTPLogsPostsEmptySlice(t *testing.T) { + called := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if len(body) != 0 { + t.Fatalf("body = %q, want empty", body) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + if err := NewVictoriaLogsSender(server.URL, "server-1").SendHTTPLogs(nil); err != nil { + t.Fatal(err) + } + if !called { + t.Fatal("SendHTTPLogs did not post the empty slice") + } +} + +func TestVictoriaSendMethodRejectsNonSuccessfulStatus(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer server.Close() + + err := NewVictoriaLogsSender(server.URL, "server-1").SendAgentLogs([]AgentLog{{Message: "hello"}}) + if err == nil || err.Error() != "unexpected status code: 502" { + t.Fatalf("error = %v, want unexpected status code: 502", err) + } +} diff --git a/agent/internal/metrics/victoria.go b/agent/internal/metrics/victoria.go index 1bedd144..5145dbb2 100644 --- a/agent/internal/metrics/victoria.go +++ b/agent/internal/metrics/victoria.go @@ -67,26 +67,7 @@ func (v *VictoriaMetricsSender) SendSystemStats(stats *health.SystemStats, colle writeGauge(&buf, "techulus_node_disk_usage_percent", serverID, stats.DiskUsagePercent, timestampMs) writeGauge(&buf, "techulus_node_disk_used_bytes", serverID, float64(stats.DiskUsedGb)*1024*1024*1024, timestampMs) - req, err := http.NewRequest("POST", v.endpoint+"/api/v1/import/prometheus", &buf) - if err != nil { - return fmt.Errorf("failed to create metrics request: %w", err) - } - req.Header.Set("Content-Type", "text/plain; version=0.0.4") - if v.username != "" { - req.SetBasicAuth(v.username, v.password) - } - - resp, err := v.client.Do(req) - if err != nil { - return fmt.Errorf("failed to send metrics: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("unexpected metrics status code: %d", resp.StatusCode) - } - - return nil + return v.postPrometheusImport(buf.Bytes(), nil) } func (v *VictoriaMetricsSender) SendAgentStats(stats *health.AgentProcessStats, collectedAt time.Time) error { diff --git a/agent/internal/metrics/victoria_test.go b/agent/internal/metrics/victoria_test.go index ff691eb2..5647e368 100644 --- a/agent/internal/metrics/victoria_test.go +++ b/agent/internal/metrics/victoria_test.go @@ -12,6 +12,54 @@ import ( "techulus/cloud-agent/internal/health" ) +func TestSendSystemStatsPostsPrometheusImport(t *testing.T) { + var gotPath string + var gotContentType string + var gotUsername string + var gotPassword string + var gotBody string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotContentType = r.Header.Get("Content-Type") + gotUsername, gotPassword, _ = r.BasicAuth() + body, _ := io.ReadAll(r.Body) + gotBody = string(body) + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + sender := NewVictoriaMetricsSender(strings.Replace(server.URL, "://", "://user:password@", 1), "server-1") + err := sender.SendSystemStats(&health.SystemStats{ + CpuUsagePercent: 1.25, + MemoryUsagePercent: 2.5, + MemoryUsedMb: 64, + DiskUsagePercent: 3.75, + DiskUsedGb: 4, + }, time.UnixMilli(1_700_000_000_000)) + if err != nil { + t.Fatalf("send system stats: %v", err) + } + + if gotPath != "/api/v1/import/prometheus" { + t.Fatalf("path = %q", gotPath) + } + if gotContentType != "text/plain; version=0.0.4" { + t.Fatalf("content type = %q", gotContentType) + } + if gotUsername != "user" || gotPassword != "password" { + t.Fatalf("Basic Auth = %q, %q", gotUsername, gotPassword) + } + lines := strings.Split(strings.TrimSpace(gotBody), "\n") + if len(lines) != 5 { + t.Fatalf("metric lines = %d, want 5:\n%s", len(lines), gotBody) + } + for _, line := range lines { + if !strings.HasSuffix(line, " 1700000000000") { + t.Fatalf("metric has wrong timestamp: %q", line) + } + } +} + func TestSendPrometheusMetricsAddsExtraLabels(t *testing.T) { var gotPath string var gotQuery string @@ -87,24 +135,20 @@ func TestSendContainerStatsAggregatesStableServiceLabels(t *testing.T) { sender := NewVictoriaMetricsSender(server.URL, "server-1") err := sender.SendContainerStats([]container.ResourceStats{ { - ContainerID: "container-a", - ServiceID: "svc-a", - DeploymentID: "dep-a", - CPUUsagePercent: 10, - MemoryUsagePercent: 1.5, - MemoryUsedBytes: 1024, - NetworkReceiveBytes: 100, - NetworkTransmitBytes: 200, + ContainerID: "container-a", + ServiceID: "svc-a", + DeploymentID: "dep-a", + CPUUsagePercent: 10, + MemoryUsagePercent: 1.5, + MemoryUsedBytes: 1024, }, { - ContainerID: "container-b", - ServiceID: "svc-a", - DeploymentID: "dep-b", - CPUUsagePercent: 20, - MemoryUsagePercent: 2.5, - MemoryUsedBytes: 2048, - NetworkReceiveBytes: 300, - NetworkTransmitBytes: 400, + ContainerID: "container-b", + ServiceID: "svc-a", + DeploymentID: "dep-b", + CPUUsagePercent: 20, + MemoryUsagePercent: 2.5, + MemoryUsedBytes: 2048, }, }, time.UnixMilli(1_700_000_000_000)) if err != nil { diff --git a/agent/internal/serverless/gateway.go b/agent/internal/serverless/gateway.go index cc81b367..bb7258e0 100644 --- a/agent/internal/serverless/gateway.go +++ b/agent/internal/serverless/gateway.go @@ -163,7 +163,7 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "service unavailable", http.StatusServiceUnavailable) return } - activityKey := serviceActivityKey(route.ServiceID) + activityKey := route.ServiceID g.beginActivity(activityKey) defer g.endActivity(activityKey, route.ServiceID, route.SleepAfterSeconds) @@ -289,10 +289,12 @@ func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, return nil, fmt.Errorf("no serverless route metadata for host %s", host) } - ready, sleepingLocalIDs, err := g.readyUpstreams(route, state) + resolution, err := g.inspectUpstreams(route, state) if err != nil { return nil, err } + ready := resolution.ready + sleepingLocalIDs := resolution.sleepingLocalIDs if len(sleepingLocalIDs) == 0 { if len(ready) > 0 { return ready, nil @@ -331,14 +333,6 @@ func (g *Gateway) resolveUpstreams(host string) ([]agenthttp.ServerlessUpstream, return g.waitForReadyUpstreams(route, route.WakeTimeoutSeconds, wakeStartedAt, startedIDs) } -func (g *Gateway) readyUpstreams(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState) ([]agenthttp.ServerlessUpstream, []string, error) { - resolution, err := g.inspectUpstreams(route, state) - if err != nil { - return nil, nil, err - } - return resolution.ready, resolution.sleepingLocalIDs, nil -} - func (g *Gateway) inspectUpstreams(route *agenthttp.ServerlessRoute, state *agenthttp.ExpectedState) (upstreamResolution, error) { actualContainers, err := g.runtime.ListServerlessContainers() if err != nil { @@ -693,10 +687,6 @@ func formatLocalUpstreamURLs(upstreams []agenthttp.ServerlessUpstream) string { } func summarizeWaitReasons(reasons []upstreamWaitReason) string { - return formatWaitReasons(reasons, true) -} - -func formatWaitReasons(reasons []upstreamWaitReason, includeDialLatency bool) string { if len(reasons) == 0 { return "" } @@ -712,7 +702,7 @@ func formatWaitReasons(reasons []upstreamWaitReason, includeDialLatency bool) st if reason.health != "" { part += fmt.Sprintf(" health=%s", reason.health) } - if includeDialLatency && reason.dialLatency > 0 { + if reason.dialLatency > 0 { part += fmt.Sprintf(" dial_latency=%s", roundDuration(reason.dialLatency)) } if reason.err != nil { @@ -888,7 +878,7 @@ func (g *Gateway) seedIdleTimers() { if !hasRunningLocalDeployment(route.LocalDeploymentIDs, actualByDeploymentID) { continue } - g.scheduleSleepTimer(serviceActivityKey(route.ServiceID), route.ServiceID, route.SleepAfterSeconds) + g.scheduleSleepTimer(route.ServiceID, route.ServiceID, route.SleepAfterSeconds) } } @@ -950,18 +940,8 @@ func (g *Gateway) endActivity(key string, serviceID string, sleepAfterSeconds in }) } -func (g *Gateway) sleepHost(host string) { - route := findRoute(g.runtime.ExpectedState(), host) - if route == nil { - log.Printf("[serverless-gateway] sleep skipped host=%s reason=missing_route_metadata", host) - return - } - g.sleepService(route.ServiceID) -} - func (g *Gateway) sleepService(serviceID string) { - activityKey := serviceActivityKey(serviceID) - activity := g.activity(activityKey) + activity := g.activity(serviceID) activity.mu.Lock() if activity.activeRequests > 0 { activeRequests := activity.activeRequests @@ -1329,10 +1309,6 @@ func sleepDelay(sleepAfterSeconds int) time.Duration { return delay } -func serviceActivityKey(serviceID string) string { - return "service:" + serviceID -} - func roundDuration(duration time.Duration) time.Duration { return duration.Round(time.Millisecond) } diff --git a/agent/internal/serverless/gateway_test.go b/agent/internal/serverless/gateway_test.go index aa07ec45..e6599a6d 100644 --- a/agent/internal/serverless/gateway_test.go +++ b/agent/internal/serverless/gateway_test.go @@ -31,7 +31,6 @@ type fakeRuntime struct { transitions []agenthttp.ServerlessTransition stopped []string deployCalls int - deployErr error afterList func() healthStatus string deployStarted chan struct{} @@ -58,9 +57,6 @@ func (f *fakeRuntime) DeployServerlessContainer(expected agenthttp.ExpectedConta f.mu.Lock() defer f.mu.Unlock() f.deployCalls += 1 - if f.deployErr != nil { - return f.deployErr - } for i, actual := range f.containers { if actual.DeploymentID == expected.DeploymentID { f.containers[i].State = "running" @@ -146,6 +142,20 @@ func (f *fakeRuntime) setState(state *agenthttp.ExpectedState, containers []cont f.containers = append([]container.Container(nil), containers...) } +func useFastWakePolling(t *testing.T) { + t.Helper() + previous := wakePollInterval + wakePollInterval = time.Millisecond + t.Cleanup(func() { wakePollInterval = previous }) +} + +func stubUpstreamReadiness(t *testing.T, check func(string) upstreamReadiness) { + t.Helper() + previous := checkUpstreamReady + checkUpstreamReady = check + t.Cleanup(func() { checkUpstreamReady = previous }) +} + func receiveProbe(t *testing.T, probes <-chan string) string { t.Helper() select { @@ -406,11 +416,7 @@ func TestRequestCanUseAlwaysOnWorkerWhileLocalDeploymentWakes(t *testing.T) { } func TestSleepingLocalDeploymentWakesAndReturnsLocalUpstream(t *testing.T) { - previousPoll := wakePollInterval - wakePollInterval = time.Millisecond - t.Cleanup(func() { - wakePollInterval = previousPoll - }) + useFastWakePolling(t) state := testExpectedState("stopped") state.Serverless.Routes[0].Upstreams = nil @@ -435,22 +441,16 @@ func TestSleepingLocalDeploymentWakesAndReturnsLocalUpstream(t *testing.T) { } func TestSleepingLocalDeploymentPrefersPublishedLoopbackUpstream(t *testing.T) { - previousPoll := wakePollInterval - previousReady := checkUpstreamReady - wakePollInterval = time.Millisecond releaseStatic := make(chan struct{}) - checkUpstreamReady = func(address string) upstreamReadiness { + stubUpstreamReadiness(t, func(address string) upstreamReadiness { if address == "127.0.0.1:31000" { return upstreamReadiness{ready: true} } <-releaseStatic return upstreamReadiness{err: errors.New("connection refused")} - } - t.Cleanup(func() { - close(releaseStatic) - wakePollInterval = previousPoll - checkUpstreamReady = previousReady }) + useFastWakePolling(t) + t.Cleanup(func() { close(releaseStatic) }) state := testExpectedState("stopped") state.Containers[0].PublishLocalPorts = true @@ -471,19 +471,13 @@ func TestSleepingLocalDeploymentPrefersPublishedLoopbackUpstream(t *testing.T) { } func TestSleepingLocalDeploymentFallsBackToStaticIPWhenLoopbackUnreachable(t *testing.T) { - previousPoll := wakePollInterval - previousReady := checkUpstreamReady - wakePollInterval = time.Millisecond - checkUpstreamReady = func(address string) upstreamReadiness { + stubUpstreamReadiness(t, func(address string) upstreamReadiness { if address == "10.0.0.10:3000" { return upstreamReadiness{ready: true} } return upstreamReadiness{err: errors.New("connection refused")} - } - t.Cleanup(func() { - wakePollInterval = previousPoll - checkUpstreamReady = previousReady }) + useFastWakePolling(t) state := testExpectedState("stopped") state.Containers[0].PublishLocalPorts = true @@ -504,22 +498,16 @@ func TestSleepingLocalDeploymentFallsBackToStaticIPWhenLoopbackUnreachable(t *te } func TestPublishedLoopbackAndStaticIPAreProbedConcurrently(t *testing.T) { - previousPoll := wakePollInterval - previousReady := checkUpstreamReady - wakePollInterval = time.Millisecond probes := make(chan string, 2) release := make(chan struct{}) var releaseOnce sync.Once - checkUpstreamReady = func(address string) upstreamReadiness { + stubUpstreamReadiness(t, func(address string) upstreamReadiness { probes <- address <-release return upstreamReadiness{err: errors.New("connection refused")} - } - t.Cleanup(func() { - releaseOnce.Do(func() { close(release) }) - wakePollInterval = previousPoll - checkUpstreamReady = previousReady }) + useFastWakePolling(t) + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) state := testExpectedState("running") state.Containers[0].PublishLocalPorts = true @@ -559,20 +547,14 @@ func TestPublishedLoopbackAndStaticIPAreProbedConcurrently(t *testing.T) { } func TestPublishedLocalPortsWithoutMatchingHostPortUsesStaticIP(t *testing.T) { - previousPoll := wakePollInterval - previousReady := checkUpstreamReady - wakePollInterval = time.Millisecond - checkUpstreamReady = func(address string) upstreamReadiness { + stubUpstreamReadiness(t, func(address string) upstreamReadiness { if address != "10.0.0.10:3000" { t.Errorf("checked %s, want only static IP", address) return upstreamReadiness{err: errors.New("unexpected upstream")} } return upstreamReadiness{ready: true} - } - t.Cleanup(func() { - wakePollInterval = previousPoll - checkUpstreamReady = previousReady }) + useFastWakePolling(t) state := testExpectedState("stopped") state.Containers[0].PublishLocalPorts = true @@ -593,11 +575,7 @@ func TestPublishedLocalPortsWithoutMatchingHostPortUsesStaticIP(t *testing.T) { } func TestSleepingLocalDeploymentStartsExistingStoppedContainer(t *testing.T) { - previousPoll := wakePollInterval - wakePollInterval = time.Millisecond - t.Cleanup(func() { - wakePollInterval = previousPoll - }) + useFastWakePolling(t) state := testExpectedState("stopped") state.Serverless.Routes[0].Upstreams = nil @@ -631,7 +609,7 @@ func TestSleepingLocalDeploymentStartsExistingStoppedContainer(t *testing.T) { } } -func TestSleepHostStopsLocalContainerAndReportsSleep(t *testing.T) { +func TestSleepServiceStopsLocalContainerAndReportsSleep(t *testing.T) { state := testExpectedState("running") runtime := &fakeRuntime{ state: state, @@ -641,7 +619,7 @@ func TestSleepHostStopsLocalContainerAndReportsSleep(t *testing.T) { } gateway := NewGateway(runtime) - gateway.sleepHost("app.example.com") + gateway.sleepService("svc_1") transitions, stopped, _ := runtime.snapshot() if len(stopped) != 1 || stopped[0] != "ctr-local" { @@ -655,7 +633,7 @@ func TestSleepHostStopsLocalContainerAndReportsSleep(t *testing.T) { } } -func TestSleepHostRechecksActivityBeforeStoppingContainer(t *testing.T) { +func TestSleepServiceRechecksActivityBeforeStoppingContainer(t *testing.T) { state := testExpectedState("running") runtime := &fakeRuntime{ state: state, @@ -665,10 +643,10 @@ func TestSleepHostRechecksActivityBeforeStoppingContainer(t *testing.T) { } gateway := NewGateway(runtime) runtime.afterList = func() { - gateway.beginActivity(serviceActivityKey("svc_1")) + gateway.beginActivity("svc_1") } - gateway.sleepHost("app.example.com") + gateway.sleepService("svc_1") transitions, stopped, _ := runtime.snapshot() if len(stopped) != 0 { @@ -679,7 +657,7 @@ func TestSleepHostRechecksActivityBeforeStoppingContainer(t *testing.T) { } } -func TestSleepHostUsesServiceActivityAcrossDomains(t *testing.T) { +func TestSleepServiceUsesServiceActivityAcrossDomains(t *testing.T) { state := testExpectedState("running") secondRoute := state.Serverless.Routes[0] secondRoute.Domain = "api.example.com" @@ -691,9 +669,9 @@ func TestSleepHostUsesServiceActivityAcrossDomains(t *testing.T) { }, } gateway := NewGateway(runtime) - gateway.beginActivity(serviceActivityKey("svc_1")) + gateway.beginActivity("svc_1") - gateway.sleepHost("api.example.com") + gateway.sleepService("svc_1") transitions, stopped, _ := runtime.snapshot() if len(stopped) != 0 { @@ -737,19 +715,13 @@ func TestPendingSleepCanWakeBeforeExpectedStateSettles(t *testing.T) { } func TestBlockingWakeRefreshesRouteAfterRedeploy(t *testing.T) { - previousPoll := wakePollInterval - previousReady := checkUpstreamReady - wakePollInterval = time.Millisecond - checkUpstreamReady = func(address string) upstreamReadiness { + stubUpstreamReadiness(t, func(address string) upstreamReadiness { if address == "10.0.0.11:3000" { return upstreamReadiness{ready: true} } return upstreamReadiness{err: errors.New("connection refused")} - } - t.Cleanup(func() { - wakePollInterval = previousPoll - checkUpstreamReady = previousReady }) + useFastWakePolling(t) oldState := testExpectedStateWithLocalDeployment("running", "dep_local", "10.0.0.10") oldState.Serverless.Routes[0].Upstreams = nil @@ -786,16 +758,10 @@ func TestBlockingWakeRefreshesRouteAfterRedeploy(t *testing.T) { } func TestBlockingWakeReturnsWhenRouteDisappearsDuringRedeploy(t *testing.T) { - previousPoll := wakePollInterval - previousReady := checkUpstreamReady - wakePollInterval = time.Millisecond - checkUpstreamReady = func(string) upstreamReadiness { + stubUpstreamReadiness(t, func(string) upstreamReadiness { return upstreamReadiness{err: errors.New("connection refused")} - } - t.Cleanup(func() { - wakePollInterval = previousPoll - checkUpstreamReady = previousReady }) + useFastWakePolling(t) oldState := testExpectedState("running") oldState.Serverless.Routes[0].Upstreams = nil @@ -836,16 +802,10 @@ func TestBlockingWakeReturnsWhenRouteDisappearsDuringRedeploy(t *testing.T) { } func TestWakeMonitorDropsStaleDeploymentAfterRedeploy(t *testing.T) { - previousPoll := wakePollInterval - previousReady := checkUpstreamReady - wakePollInterval = time.Millisecond - checkUpstreamReady = func(string) upstreamReadiness { + stubUpstreamReadiness(t, func(string) upstreamReadiness { return upstreamReadiness{err: errors.New("connection refused")} - } - t.Cleanup(func() { - wakePollInterval = previousPoll - checkUpstreamReady = previousReady }) + useFastWakePolling(t) oldState := testExpectedStateWithLocalDeployment("running", "dep_local", "10.0.0.10") oldState.Serverless.Routes[0].Upstreams = nil @@ -885,18 +845,11 @@ func TestWakeMonitorDropsStaleDeploymentAfterRedeploy(t *testing.T) { } func TestWakeTimeoutQueuesWakeFailedTransition(t *testing.T) { - previousPoll := wakePollInterval - wakePollInterval = time.Millisecond - t.Cleanup(func() { - wakePollInterval = previousPoll - }) + useFastWakePolling(t) state := testExpectedState("stopped") state.Containers[0].HealthCheck = &agenthttp.HealthCheck{ - Cmd: "curl http://localhost:3000/health", - Interval: 1, - Timeout: 1, - Retries: 1, + Cmd: "curl http://localhost:3000/health", } state.Serverless.Routes[0].Upstreams = nil state.Serverless.Routes[0].WakeTimeoutSeconds = 1 @@ -927,16 +880,10 @@ func TestWakeTimeoutQueuesWakeFailedTransition(t *testing.T) { } func TestWakeTimeoutWhenLocalPortNeverOpens(t *testing.T) { - previousPoll := wakePollInterval - previousReady := checkUpstreamReady - wakePollInterval = time.Millisecond - checkUpstreamReady = func(string) upstreamReadiness { + stubUpstreamReadiness(t, func(string) upstreamReadiness { return upstreamReadiness{err: errors.New("connection refused")} - } - t.Cleanup(func() { - wakePollInterval = previousPoll - checkUpstreamReady = previousReady }) + useFastWakePolling(t) var logs bytes.Buffer previousOutput := log.Writer() @@ -980,10 +927,7 @@ func testExpectedState(localDesiredState string) *agenthttp.ExpectedState { { DeploymentID: "dep_local", ServiceID: "svc_1", - ServiceName: "api", - Name: "svc_1-dep_local", DesiredState: localDesiredState, - Image: "nginx", IPAddress: "10.0.0.10", }, }, @@ -999,7 +943,6 @@ func testExpectedState(localDesiredState string) *agenthttp.ExpectedState { Upstreams: []agenthttp.ServerlessUpstream{ { DeploymentID: "dep_worker", - ServerID: "server_worker", Url: "10.0.0.20:3000", AlwaysOn: true, }, @@ -1012,7 +955,6 @@ func testExpectedState(localDesiredState string) *agenthttp.ExpectedState { func testExpectedStateWithLocalDeployment(localDesiredState string, deploymentID string, ipAddress string) *agenthttp.ExpectedState { state := testExpectedState(localDesiredState) state.Containers[0].DeploymentID = deploymentID - state.Containers[0].Name = "svc_1-" + deploymentID state.Containers[0].IPAddress = ipAddress state.Serverless.Routes[0].LocalDeploymentIDs = []string{deploymentID} return state diff --git a/agent/internal/traefik/static.go b/agent/internal/traefik/static.go index 9fa007dd..6c9cf2bd 100644 --- a/agent/internal/traefik/static.go +++ b/agent/internal/traefik/static.go @@ -63,18 +63,7 @@ func validateStaticConfig(data []byte) error { return nil } -func EnsureEntryPoints(tcpPorts []int, udpPorts []int) (needsRestart bool, err error) { - for _, port := range tcpPorts { - if err := ValidateTCPPort(port); err != nil { - return false, fmt.Errorf("invalid entry point: %w", err) - } - } - for _, port := range udpPorts { - if err := ValidateUDPPort(port); err != nil { - return false, fmt.Errorf("invalid entry point: %w", err) - } - } - +func updateStaticConfig(mutate func(map[string]interface{}) bool) (bool, error) { originalData, err := os.ReadFile(traefikStaticConfigPath) if err != nil { return false, fmt.Errorf("failed to read static config: %w", err) @@ -85,37 +74,7 @@ func EnsureEntryPoints(tcpPorts []int, udpPorts []int) (needsRestart bool, err e return false, fmt.Errorf("failed to parse static config: %w", err) } - entryPoints, ok := config["entryPoints"].(map[string]interface{}) - if !ok { - entryPoints = make(map[string]interface{}) - config["entryPoints"] = entryPoints - } - - modified := false - - for _, port := range tcpPorts { - name := fmt.Sprintf("tcp-%d", port) - if _, exists := entryPoints[name]; !exists { - entryPoints[name] = map[string]interface{}{ - "address": fmt.Sprintf(":%d", port), - } - modified = true - log.Printf("[traefik] adding TCP entry point: %s", name) - } - } - - for _, port := range udpPorts { - name := fmt.Sprintf("udp-%d", port) - if _, exists := entryPoints[name]; !exists { - entryPoints[name] = map[string]interface{}{ - "address": fmt.Sprintf(":%d/udp", port), - } - modified = true - log.Printf("[traefik] adding UDP entry point: %s", name) - } - } - - if !modified { + if !mutate(config) { return false, nil } @@ -132,36 +91,64 @@ func EnsureEntryPoints(tcpPorts []int, udpPorts []int) (needsRestart bool, err e return false, fmt.Errorf("failed to write static config: %w", err) } - log.Printf("[traefik] static config updated, restart required") return true, nil } -func EnsureMetricsConfig() (needsRestart bool, err error) { - originalData, err := os.ReadFile(traefikStaticConfigPath) - if err != nil { - return false, fmt.Errorf("failed to read static config: %w", err) +func EnsureEntryPoints(tcpPorts []int, udpPorts []int) (needsRestart bool, err error) { + for _, port := range tcpPorts { + if err := ValidateTCPPort(port); err != nil { + return false, fmt.Errorf("invalid entry point: %w", err) + } } - - var config map[string]interface{} - if err := yaml.Unmarshal(originalData, &config); err != nil { - return false, fmt.Errorf("failed to parse static config: %w", err) + for _, port := range udpPorts { + if err := ValidateUDPPort(port); err != nil { + return false, fmt.Errorf("invalid entry point: %w", err) + } } - if !ensurePrometheusMetricsConfig(config) { - return false, nil - } + needsRestart, err = updateStaticConfig(func(config map[string]interface{}) bool { + entryPoints, ok := config["entryPoints"].(map[string]interface{}) + if !ok { + entryPoints = make(map[string]interface{}) + config["entryPoints"] = entryPoints + } - newData, err := yaml.Marshal(config) - if err != nil { - return false, fmt.Errorf("failed to marshal static config: %w", err) - } + modified := false + for _, port := range tcpPorts { + name := fmt.Sprintf("tcp-%d", port) + if _, exists := entryPoints[name]; !exists { + entryPoints[name] = map[string]interface{}{ + "address": fmt.Sprintf(":%d", port), + } + modified = true + log.Printf("[traefik] adding TCP entry point: %s", name) + } + } - if err := validateStaticConfig(newData); err != nil { - return false, fmt.Errorf("config validation failed: %w", err) + for _, port := range udpPorts { + name := fmt.Sprintf("udp-%d", port) + if _, exists := entryPoints[name]; !exists { + entryPoints[name] = map[string]interface{}{ + "address": fmt.Sprintf(":%d/udp", port), + } + modified = true + log.Printf("[traefik] adding UDP entry point: %s", name) + } + } + return modified + }) + if err != nil || !needsRestart { + return needsRestart, err } - if err := atomicWrite(traefikStaticConfigPath, newData, 0644); err != nil { - return false, fmt.Errorf("failed to write static config: %w", err) + log.Printf("[traefik] static config updated, restart required") + return true, nil +} + +func EnsureMetricsConfig() (needsRestart bool, err error) { + needsRestart, err = updateStaticConfig(ensurePrometheusMetricsConfig) + if err != nil || !needsRestart { + return needsRestart, err } log.Printf("[traefik] metrics config updated, restart required") diff --git a/cli/internal/api/client.go b/cli/internal/api/client.go index 8dedbf33..8cdaa68d 100644 --- a/cli/internal/api/client.go +++ b/cli/internal/api/client.go @@ -75,7 +75,7 @@ func (c *Client) RequestJSON(ctx context.Context, method, path string, query url return JSON(ctx, c.HTTPClient, method, endpoint, headers, body, out) } -func JSON(ctx context.Context, client *http.Client, method, endpoint string, headers map[string]string, body any, out any) error { +func requestJSON(ctx context.Context, client *http.Client, method, endpoint string, headers map[string]string, body any) (int, []byte, error) { if client == nil { client = http.DefaultClient } @@ -83,13 +83,13 @@ func JSON(ctx context.Context, client *http.Client, method, endpoint string, hea if body != nil { raw, err := json.Marshal(body) if err != nil { - return err + return 0, nil, err } reader = bytes.NewReader(raw) } req, err := http.NewRequestWithContext(ctx, method, endpoint, reader) if err != nil { - return err + return 0, nil, err } req.Header.Set("content-type", "application/json") for key, value := range headers { @@ -98,15 +98,20 @@ func JSON(ctx context.Context, client *http.Client, method, endpoint string, hea resp, err := client.Do(req) if err != nil { - return err + return 0, nil, err } defer resp.Body.Close() raw, err := io.ReadAll(resp.Body) + return resp.StatusCode, raw, err +} + +func JSON(ctx context.Context, client *http.Client, method, endpoint string, headers map[string]string, body any, out any) error { + status, raw, err := requestJSON(ctx, client, method, endpoint, headers, body) if err != nil { return err } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { + if status < 200 || status >= 300 { var apiErr ErrorResponse _ = json.Unmarshal(raw, &apiErr) message := apiErr.Message @@ -119,7 +124,7 @@ func JSON(ctx context.Context, client *http.Client, method, endpoint string, hea host = NormalizeHost(parsed.Scheme + "://" + parsed.Host) } return &APIError{ - Status: resp.StatusCode, + Status: status, Message: message, Code: apiErr.Code, Host: host, @@ -135,36 +140,14 @@ func JSON(ctx context.Context, client *http.Client, method, endpoint string, hea } func JSONStatus(ctx context.Context, client *http.Client, method, endpoint string, body any, out any) (int, error) { - if client == nil { - client = http.DefaultClient - } - var reader io.Reader - if body != nil { - raw, err := json.Marshal(body) - if err != nil { - return 0, err - } - reader = bytes.NewReader(raw) - } - req, err := http.NewRequestWithContext(ctx, method, endpoint, reader) - if err != nil { - return 0, err - } - req.Header.Set("content-type", "application/json") - - resp, err := client.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - raw, err := io.ReadAll(resp.Body) + status, raw, err := requestJSON(ctx, client, method, endpoint, nil, body) if err != nil { - return resp.StatusCode, err + return status, err } if len(raw) > 0 && out != nil { if err := json.Unmarshal(raw, out); err != nil { - return resp.StatusCode, fmt.Errorf("invalid JSON response: %w", err) + return status, fmt.Errorf("invalid JSON response: %w", err) } } - return resp.StatusCode, nil + return status, nil } diff --git a/cli/internal/api/client_test.go b/cli/internal/api/client_test.go index 770006c8..5e71c582 100644 --- a/cli/internal/api/client_test.go +++ b/cli/internal/api/client_test.go @@ -2,6 +2,8 @@ package api import ( "context" + "errors" + "io" "net/http" "net/http/httptest" "strings" @@ -34,14 +36,15 @@ func TestRequestJSONSuccess(t *testing.T) { func TestRequestJSONAPIError(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(`{"error":"Bad manifest"}`)) + w.Write([]byte(`{"error":"Bad manifest","code":"BAD_MANIFEST"}`)) })) defer server.Close() client := NewClient(server.URL, "") client.HTTPClient = server.Client() err := client.RequestJSON(context.Background(), http.MethodPost, "/test", nil, nil, nil) - if err == nil || err.Error() != "Bad manifest" { + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.Status != http.StatusBadRequest || apiErr.Message != "Bad manifest" || apiErr.Code != "BAD_MANIFEST" || apiErr.Host != server.URL { t.Fatalf("error = %v", err) } } @@ -75,3 +78,55 @@ func TestRequestJSONInvalidJSON(t *testing.T) { t.Fatalf("error = %v", err) } } + +func TestJSONSharedRequestMechanics(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatal(err) + } + if r.Method != http.MethodPost || r.Header.Get("content-type") != "application/json" || r.Header.Get("x-test") != "present" || string(body) != `{"hello":"world"}` { + t.Fatalf("request = %s content-type=%q x-test=%q body=%s", r.Method, r.Header.Get("content-type"), r.Header.Get("x-test"), body) + } + w.Write([]byte(`{"ok":true}`)) + })) + defer server.Close() + + var got struct { + OK bool `json:"ok"` + } + if err := JSON(context.Background(), nil, http.MethodPost, server.URL, map[string]string{"x-test": "present"}, map[string]string{"hello": "world"}, &got); err != nil || !got.OK { + t.Fatalf("JSON() result = %#v, error = %v", got, err) + } +} + +func TestJSONStatusDecodesNon2xx(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error":"authorization_pending","error_description":"waiting"}`)) + })) + defer server.Close() + + var got struct { + Error string `json:"error"` + ErrorDescription string `json:"error_description"` + } + status, err := JSONStatus(context.Background(), server.Client(), http.MethodPost, server.URL, nil, &got) + if err != nil || status != http.StatusBadRequest || got.Error != "authorization_pending" || got.ErrorDescription != "waiting" { + t.Fatalf("status = %d, response = %#v, error = %v", status, got, err) + } +} + +func TestJSONStatusPreservesStatusOnReadError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-length", "100") + w.WriteHeader(http.StatusTeapot) + w.Write([]byte(`{}`)) + })) + defer server.Close() + + status, err := JSONStatus(context.Background(), server.Client(), http.MethodGet, server.URL, nil, nil) + if status != http.StatusTeapot || !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("status = %d, error = %v", status, err) + } +} diff --git a/cli/internal/auth/config.go b/cli/internal/auth/config.go index 85d23154..91bf2c4d 100644 --- a/cli/internal/auth/config.go +++ b/cli/internal/auth/config.go @@ -8,7 +8,25 @@ import ( "runtime" ) -const ConfigDirName = "techulus-cloud-cli" +const ( + ConfigDirName = "techulus-cloud-cli" + DevConfigDirName = "techulus-cloud-cli-dev" +) + +type ConfigStore struct { + development bool +} + +func NewConfigStore(version string) ConfigStore { + return ConfigStore{development: version == "dev"} +} + +func (s ConfigStore) dirName() string { + if s.development { + return DevConfigDirName + } + return ConfigDirName +} type Config struct { Host string `json:"host"` @@ -41,20 +59,20 @@ func ConfigRootFor(goos, home string, getenv func(string) string) string { return filepath.Join(home, ".config") } -func ConfigDir() string { - return filepath.Join(ConfigRoot(), ConfigDirName) +func (s ConfigStore) ConfigDir() string { + return filepath.Join(ConfigRoot(), s.dirName()) } -func ConfigPath() string { - return filepath.Join(ConfigDir(), "config.json") +func (s ConfigStore) ConfigPath() string { + return filepath.Join(s.ConfigDir(), "config.json") } -func ConfigPathFor(goos, home string, getenv func(string) string) string { - return filepath.Join(ConfigRootFor(goos, home, getenv), ConfigDirName, "config.json") +func (s ConfigStore) ConfigPathFor(goos, home string, getenv func(string) string) string { + return filepath.Join(ConfigRootFor(goos, home, getenv), s.dirName(), "config.json") } -func ReadConfig() (*Config, error) { - contents, err := os.ReadFile(ConfigPath()) +func (s ConfigStore) ReadConfig() (*Config, error) { + contents, err := os.ReadFile(s.ConfigPath()) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, nil @@ -69,22 +87,22 @@ func ReadConfig() (*Config, error) { return &config, nil } -func WriteConfig(config Config) error { - if err := os.MkdirAll(ConfigDir(), 0o700); err != nil { +func (s ConfigStore) WriteConfig(config Config) error { + if err := os.MkdirAll(s.ConfigDir(), 0o700); err != nil { return err } contents, err := json.MarshalIndent(config, "", "\t") if err != nil { return err } - if err := os.WriteFile(ConfigPath(), contents, 0o600); err != nil { + if err := os.WriteFile(s.ConfigPath(), contents, 0o600); err != nil { return err } - return os.Chmod(ConfigPath(), 0o600) + return os.Chmod(s.ConfigPath(), 0o600) } -func DeleteConfig() error { - err := os.Remove(ConfigPath()) +func (s ConfigStore) DeleteConfig() error { + err := os.Remove(s.ConfigPath()) if errors.Is(err, os.ErrNotExist) { return nil } diff --git a/cli/internal/auth/config_test.go b/cli/internal/auth/config_test.go index 1af7d33f..20a3d294 100644 --- a/cli/internal/auth/config_test.go +++ b/cli/internal/auth/config_test.go @@ -7,10 +7,11 @@ import ( func TestConfigPathForPlatforms(t *testing.T) { tests := []struct { - name string - goos string - env map[string]string - want string + name string + version string + goos string + env map[string]string + want string }{ { name: "darwin", @@ -28,6 +29,13 @@ func TestConfigPathForPlatforms(t *testing.T) { goos: "linux", want: filepath.Join("/home/alice", ".config", ConfigDirName, "config.json"), }, + { + name: "development", + version: "dev", + goos: "linux", + env: map[string]string{"XDG_CONFIG_HOME": "/xdg"}, + want: filepath.Join("/xdg", DevConfigDirName, "config.json"), + }, { name: "windows appdata", goos: "windows", @@ -38,12 +46,99 @@ func TestConfigPathForPlatforms(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := ConfigPathFor(tt.goos, "/home/alice", func(key string) string { + store := NewConfigStore(tt.version) + got := store.ConfigPathFor(tt.goos, "/home/alice", func(key string) string { return tt.env[key] }) if got != tt.want { - t.Fatalf("ConfigPathFor() = %q, want %q", got, tt.want) + t.Fatalf("ConfigStore.ConfigPathFor() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestConfigStoreVersionClassification(t *testing.T) { + tests := []struct { + name string + store ConfigStore + wantDir string + }{ + {name: "development", store: NewConfigStore("dev"), wantDir: DevConfigDirName}, + {name: "release", store: NewConfigStore("v1.0.0"), wantDir: ConfigDirName}, + {name: "empty version", store: NewConfigStore(""), wantDir: ConfigDirName}, + {name: "development suffix", store: NewConfigStore("dev-dirty"), wantDir: ConfigDirName}, + {name: "zero value", store: ConfigStore{}, wantDir: ConfigDirName}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.store.ConfigPathFor("linux", "/home/alice", func(string) string { return "" }) + want := filepath.Join("/home/alice", ".config", tt.wantDir, "config.json") + if got != want { + t.Fatalf("ConfigStore.ConfigPathFor() = %q, want %q", got, want) } }) } } + +func TestDevelopmentAndProductionConfigsAreIndependent(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + + production := NewConfigStore("v1.0.0") + development := NewConfigStore("dev") + if production.ConfigPath() == development.ConfigPath() { + t.Fatalf("production and development config paths are both %q", production.ConfigPath()) + } + + if err := production.WriteConfig(Config{Host: "https://cloud.example.com", APIKey: "production"}); err != nil { + t.Fatal(err) + } + if err := development.WriteConfig(Config{Host: "http://localhost:3000", APIKey: "development"}); err != nil { + t.Fatal(err) + } + + productionConfig, err := production.ReadConfig() + if err != nil { + t.Fatal(err) + } + developmentConfig, err := development.ReadConfig() + if err != nil { + t.Fatal(err) + } + if productionConfig.APIKey != "production" || developmentConfig.APIKey != "development" { + t.Fatalf("production config = %#v, development config = %#v", productionConfig, developmentConfig) + } + + if err := development.DeleteConfig(); err != nil { + t.Fatal(err) + } + developmentConfig, err = development.ReadConfig() + if err != nil { + t.Fatal(err) + } + productionConfig, err = production.ReadConfig() + if err != nil { + t.Fatal(err) + } + if developmentConfig != nil || productionConfig == nil || productionConfig.APIKey != "production" { + t.Fatalf("after development delete, production config = %#v, development config = %#v", productionConfig, developmentConfig) + } + + if err := development.WriteConfig(Config{Host: "http://localhost:3000", APIKey: "development"}); err != nil { + t.Fatal(err) + } + if err := production.DeleteConfig(); err != nil { + t.Fatal(err) + } + productionConfig, err = production.ReadConfig() + if err != nil { + t.Fatal(err) + } + developmentConfig, err = development.ReadConfig() + if err != nil { + t.Fatal(err) + } + if productionConfig != nil || developmentConfig == nil || developmentConfig.APIKey != "development" { + t.Fatalf("after production delete, production config = %#v, development config = %#v", productionConfig, developmentConfig) + } +} diff --git a/cli/internal/cli/app.go b/cli/internal/cli/app.go index fd089775..9cfdedd6 100644 --- a/cli/internal/cli/app.go +++ b/cli/internal/cli/app.go @@ -12,6 +12,7 @@ import ( "os/signal" "path/filepath" "runtime" + "slices" "strconv" "strings" "time" @@ -34,15 +35,14 @@ const ( type App struct { Version string - Args []string In io.Reader Out io.Writer Err io.Writer HTTPClient *http.Client Sleep func(time.Duration) - Now func() time.Time IsInteractive func() bool GetCWD func() (string, error) + configStore auth.ConfigStore flags globalFlags } @@ -80,7 +80,6 @@ func NewApp(version string, in io.Reader, out io.Writer, errOut io.Writer) *App Out: out, Err: errOut, HTTPClient: &http.Client{Timeout: defaultAPITimeout}, - Now: time.Now, IsInteractive: func() bool { inFile, inOK := in.(*os.File) outFile, outOK := out.(*os.File) @@ -99,6 +98,7 @@ func NewApp(version string, in io.Reader, out io.Writer, errOut io.Writer) *App } return os.Getwd() }, + configStore: auth.NewConfigStore(version), } } @@ -107,14 +107,11 @@ func (a *App) Execute() error { cmd.SetIn(a.In) cmd.SetOut(a.Out) cmd.SetErr(a.Err) - if a.Args != nil { - cmd.SetArgs(a.Args) - } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() if err := cmd.ExecuteContext(ctx); err != nil { if a.isMachineOutput() { - _ = a.writeError(err) + _ = output.Error(a.Out, err) return handledError{err: err} } return err @@ -197,7 +194,7 @@ func (a *App) authLoginCommand() *cobra.Command { return errors.New("tc auth login requires human browser approval and does not support --agent or --json") } if host == "" { - existing, err := auth.ReadConfig() + existing, err := a.configStore.ReadConfig() if err != nil { return err } @@ -220,7 +217,7 @@ func (a *App) authLogoutCommand() *cobra.Command { Use: "logout", Short: "Remove the saved CLI session", RunE: func(cmd *cobra.Command, args []string) error { - if err := auth.DeleteConfig(); err != nil { + if err := a.configStore.DeleteConfig(); err != nil { return err } if a.isMachineOutput() { @@ -238,7 +235,7 @@ func (a *App) authWhoamiCommand() *cobra.Command { Use: "whoami", Short: "Show the current CLI account", RunE: func(cmd *cobra.Command, args []string) error { - config, err := requireConfig() + config, err := a.requireConfig() if err != nil { return err } @@ -300,6 +297,8 @@ service: image: nginx:1.27 hostname: null replicas: 1 + placement: + mode: automatic healthCheck: null startCommand: null ports: @@ -345,7 +344,7 @@ func (a *App) linkCommand() *cobra.Command { if explicitIDs == 0 && !a.IsInteractive() { return errors.New("tc link requires an interactive terminal or all ID flags") } - config, err := requireConfig() + config, err := a.requireConfig() if err != nil { return err } @@ -380,7 +379,7 @@ func (a *App) linkCommand() *cobra.Command { return errors.New("project ID not found") } } else { - project, err = selectFromList(reader, a.Out, "Select a project:", ps.Projects, func(v projectItem) string { return v.Name }, nil) + project, err = selectFromList(reader, a.Out, "Select a project:", ps.Projects, func(v projectItem) string { return v.Name }) if err != nil { return err } @@ -401,7 +400,7 @@ func (a *App) linkCommand() *cobra.Command { return errors.New("environment ID not found") } } else { - environment, err = selectFromList(reader, a.Out, "Select an environment:", es.Environments, func(v environmentItem) string { return v.Name }, nil) + environment, err = selectFromList(reader, a.Out, "Select an environment:", es.Environments, func(v environmentItem) string { return v.Name }) if err != nil { return err } @@ -422,7 +421,7 @@ func (a *App) linkCommand() *cobra.Command { return errors.New("service ID not found") } } else { - service, err = selectFromList(reader, a.Out, "Select a service:", ss.Services, func(v serviceItem) string { return v.Name }, nil) + service, err = selectFromList(reader, a.Out, "Select a service:", ss.Services, func(v serviceItem) string { return v.Name }) if err != nil { return err } @@ -435,7 +434,14 @@ func (a *App) linkCommand() *cobra.Command { IsPublic bool `json:"public"` Domain *string `json:"domain"` } `json:"ports"` - Replicas int `json:"replicas"` + Replicas int `json:"replicas"` + Placement *struct { + Mode string `json:"mode"` + } `json:"placement"` + Placements []struct { + ServerID string `json:"serverId"` + Count int `json:"count"` + } `json:"placements"` HealthCheck *manifest.HealthCheck `json:"healthCheck"` StartCommand *string `json:"startCommand"` Resources *manifest.Resources `json:"resources"` @@ -468,7 +474,21 @@ func (a *App) linkCommand() *cobra.Command { for i, p := range cfg.Current.Ports { ports[i] = manifest.Port{ContainerPort: p.ContainerPort, Public: p.IsPublic, Domain: p.Domain} } - m := manifest.Manifest{APIVersion: "v1", Project: manifest.Project{ID: project.ID, Slug: project.Slug}, Environment: manifest.Environment{ID: environment.ID, Name: environment.Name}, Service: manifest.Service{ID: service.ID, Name: service.Name, Source: service.Source, Hostname: cfg.Current.Hostname, Ports: ports, Replicas: cfg.Current.Replicas, HealthCheck: cfg.Current.HealthCheck, StartCommand: cfg.Current.StartCommand, Resources: cfg.Current.Resources}} + var placement *manifest.Placement + if cfg.Current.Placement != nil { + if cfg.Current.Placement.Mode == "automatic" { + placement = &manifest.Placement{Mode: "automatic"} + } else if len(cfg.Current.Placements) == 0 { + return errors.New("configure at least one server placement in the web UI before linking this service") + } else { + placement = &manifest.Placement{Mode: "manual"} + placement.Servers = make([]manifest.PlacementServer, len(cfg.Current.Placements)) + for i, p := range cfg.Current.Placements { + placement.Servers[i] = manifest.PlacementServer{ServerID: p.ServerID, Count: p.Count} + } + } + } + m := manifest.Manifest{APIVersion: "v1", Project: manifest.Project{ID: project.ID, Slug: project.Slug}, Environment: manifest.Environment{ID: environment.ID, Name: environment.Name}, Service: manifest.Service{ID: service.ID, Name: service.Name, Source: service.Source, Hostname: cfg.Current.Hostname, Ports: ports, Replicas: cfg.Current.Replicas, Placement: placement, HealthCheck: cfg.Current.HealthCheck, StartCommand: cfg.Current.StartCommand, Resources: cfg.Current.Resources}} if err := manifest.Save(manifestPath, m); err != nil { return err } @@ -494,7 +514,7 @@ func (a *App) applyCommand() *cobra.Command { "agent_notes": "Requires techulus.yml in the current directory and sends the full desired manifest to the control plane.", }, RunE: func(cmd *cobra.Command, args []string) error { - config, err := requireConfig() + config, err := a.requireConfig() if err != nil { return err } @@ -507,7 +527,16 @@ func (a *App) applyCommand() *cobra.Command { if !loaded.Manifest.Linked() { return errors.New("service is not linked: run `tc link`") } - body := map[string]any{"source": sourcePatch(loaded.Manifest.Service.Source), "hostname": loaded.Manifest.Service.Hostname, "ports": loaded.Manifest.Service.Ports, "replicas": loaded.Manifest.Service.Replicas, "healthCheck": loaded.Manifest.Service.HealthCheck, "startCommand": loaded.Manifest.Service.StartCommand} + placement := loaded.Manifest.Service.Placement + if placement == nil { + return errors.New("service.placement is required") + } + body := map[string]any{"source": sourcePatch(loaded.Manifest.Service.Source), "hostname": loaded.Manifest.Service.Hostname, "ports": loaded.Manifest.Service.Ports, "healthCheck": loaded.Manifest.Service.HealthCheck, "startCommand": loaded.Manifest.Service.StartCommand} + if placement.Mode == "automatic" { + body["placement"] = map[string]any{"mode": "automatic", "replicas": loaded.Manifest.Service.Replicas} + } else { + body["placement"] = map[string]any{"mode": "manual", "placements": placement.Servers} + } if loaded.Manifest.Service.Resources != nil { body["resources"] = loaded.Manifest.Service.Resources } @@ -531,7 +560,7 @@ func (a *App) deployCommand() *cobra.Command { "agent_notes": "Requires techulus.yml in the current directory and queues a deployment for that service.", }, RunE: func(cmd *cobra.Command, args []string) error { - config, err := requireConfig() + config, err := a.requireConfig() if err != nil { return err } @@ -585,7 +614,7 @@ func (a *App) statusCommand() *cobra.Command { "agent_notes": "Without explicit target flags, tc reads techulus.yml from the current directory.\nFor agent use outside a linked directory, pass --project, --environment, and --service together.", }, RunE: func(cmd *cobra.Command, args []string) error { - config, err := requireConfig() + config, err := a.requireConfig() if err != nil { return err } @@ -630,7 +659,7 @@ func (a *App) logsCommand() *cobra.Command { } follow = false } - config, err := requireConfig() + config, err := a.requireConfig() if err != nil { return err } @@ -638,7 +667,7 @@ func (a *App) logsCommand() *cobra.Command { if err != nil { return err } - if logRange != "" && !contains([]string{"1h", "6h", "24h", "7d"}, logRange) { + if logRange != "" && !slices.Contains([]string{"1h", "6h", "24h", "7d"}, logRange) { return errors.New("invalid log range") } return a.runLogs(cmd.Context(), config, value, tail, follow, query, logRange) @@ -658,7 +687,7 @@ func (a *App) projectsCommand() *cobra.Command { Short: "List projects", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - cfg, err := requireConfig() + cfg, err := a.requireConfig() if err != nil { return err } @@ -681,7 +710,7 @@ func (a *App) projectsCommand() *cobra.Command { func (a *App) environmentsCommand() *cobra.Command { var id string c := &cobra.Command{Use: "environments", Short: "List project environments", RunE: func(cmd *cobra.Command, args []string) error { - cfg, e := requireConfig() + cfg, e := a.requireConfig() if e != nil { return e } @@ -712,7 +741,7 @@ func (a *App) environmentsCommand() *cobra.Command { func (a *App) servicesCommand() *cobra.Command { var p, eid string c := &cobra.Command{Use: "services", Short: "List environment services", RunE: func(cmd *cobra.Command, args []string) error { - cfg, e := requireConfig() + cfg, e := a.requireConfig() if e != nil { return e } @@ -750,7 +779,7 @@ func (a *App) servicesCommand() *cobra.Command { func (a *App) resourceCommand(name, short, suffix string, q func(*cobra.Command) url.Values, print func(io.Writer, map[string]any)) *cobra.Command { var target serviceTargetFlags c := &cobra.Command{Use: name, Short: short, RunE: func(cmd *cobra.Command, args []string) error { - cfg, e := requireConfig() + cfg, e := a.requireConfig() if e != nil { return e } @@ -821,7 +850,7 @@ func (a *App) rolloutCommand() *cobra.Command { return c } func (a *App) getRolloutResource(cmd *cobra.Command, t serviceTargetFlags, id string, logs bool, q string, limit int) error { - cfg, e := requireConfig() + cfg, e := a.requireConfig() if e != nil { return e } @@ -857,7 +886,7 @@ func (a *App) metricsCommand() *cobra.Command { c := a.resourceCommand("metrics", "Show service metrics", "/metrics", func(*cobra.Command) url.Values { return url.Values{"range": {r}} }, printMetrics) c.Flags().StringVar(&r, "range", "1h", "Range: 1h, 6h, 24h, 7d, 30d") c.PreRunE = func(*cobra.Command, []string) error { - if !contains([]string{"1h", "6h", "24h", "7d", "30d"}, r) { + if !slices.Contains([]string{"1h", "6h", "24h", "7d", "30d"}, r) { return errors.New("invalid metrics range") } return nil @@ -876,15 +905,6 @@ func (a *App) revisionsCommand() *cobra.Command { c.Flags().StringVar(&cursor, "cursor", "", "Pagination cursor") return c } -func contains(v []string, s string) bool { - for _, x := range v { - if x == s { - return true - } - } - return false -} - func (a *App) versionCommand() *cobra.Command { return &cobra.Command{ Use: "version", @@ -958,10 +978,6 @@ func (a *App) writeRaw(data any) error { return output.JSON(a.Out, data) } -func (a *App) writeError(err error) error { - return output.Error(a.Out, err) -} - type agentHelpInfo struct { Command string `json:"command"` Path string `json:"path"` @@ -1057,38 +1073,15 @@ func parseAgentArgs(cmd *cobra.Command) []agentArg { continue } arg := agentArg{Name: name, Required: required} - if choices := parseAgentArgChoices(name); len(choices) > 0 { - arg.Name = agentChoiceArgName(cmd) - arg.Choices = choices + if strings.Contains(name, "|") { + arg.Name = "shell" + arg.Choices = strings.Split(name, "|") } args = append(args, arg) } return args } -func parseAgentArgChoices(name string) []string { - if !strings.Contains(name, "|") { - return nil - } - parts := strings.Split(name, "|") - choices := make([]string, 0, len(parts)) - for _, part := range parts { - part = strings.TrimSpace(part) - if part == "" { - return nil - } - choices = append(choices, part) - } - return choices -} - -func agentChoiceArgName(cmd *cobra.Command) string { - if cmd.Name() == "completion" { - return "shell" - } - return "value" -} - type serviceTargetFlags struct { Project string Environment string @@ -1237,8 +1230,8 @@ func sourcesEqual(expected, actual manifest.Source) bool { return *expected.RootDir == *actual.RootDir } -func requireConfig() (*auth.Config, error) { - config, err := auth.ReadConfig() +func (a *App) requireConfig() (*auth.Config, error) { + config, err := a.configStore.ReadConfig() if err != nil { return nil, err } @@ -1271,16 +1264,16 @@ func (a *App) runAuthLogin(ctx context.Context, host string) error { if interval <= 0 { interval = 5 * time.Second } - expiresAt := a.Now().Add(time.Duration(deviceCode.ExpiresIn) * time.Second) + expiresAt := time.Now().Add(time.Duration(deviceCode.ExpiresIn) * time.Second) var accessToken string for accessToken == "" { - if deviceCode.ExpiresIn > 0 && !a.Now().Before(expiresAt) { + if deviceCode.ExpiresIn > 0 && !time.Now().Before(expiresAt) { return errors.New("device authorization expired") } if err := a.sleep(ctx, interval); err != nil { return err } - if deviceCode.ExpiresIn > 0 && !a.Now().Before(expiresAt) { + if deviceCode.ExpiresIn > 0 && !time.Now().Before(expiresAt) { return errors.New("device authorization expired") } var tokenResponse deviceTokenResponse @@ -1333,7 +1326,7 @@ func (a *App) runAuthLogin(ctx context.Context, host string) error { }, &exchange); err != nil { return err } - if err := auth.WriteConfig(auth.Config{ + if err := a.configStore.WriteConfig(auth.Config{ Host: host, APIKey: exchange.APIKey, KeyID: exchange.KeyID, @@ -2088,7 +2081,6 @@ func selectFromList[T any]( title string, items []T, render func(T) string, - disabledReason func(T) string, ) (T, error) { var zero T if len(items) == 0 { @@ -2113,16 +2105,6 @@ func selectFromList[T any]( } continue } - selected := items[choice-1] - if disabledReason != nil { - if reason := disabledReason(selected); reason != "" { - fmt.Fprintln(out, reason) - if errors.Is(err, io.EOF) { - return zero, io.ErrUnexpectedEOF - } - continue - } - } - return selected, nil + return items[choice-1], nil } } diff --git a/cli/internal/cli/app_test.go b/cli/internal/cli/app_test.go index c4b691a4..78aafe46 100644 --- a/cli/internal/cli/app_test.go +++ b/cli/internal/cli/app_test.go @@ -26,14 +26,44 @@ service: name: web source: {type: image, image: nginx:1.27} replicas: 2 + placement: {mode: automatic} hostname: null healthCheck: null startCommand: null ports: [] ` +func TestAppUsesVersionSpecificConfig(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + production := auth.NewConfigStore("v1.0.0") + development := auth.NewConfigStore("dev") + if err := production.WriteConfig(auth.Config{Host: "https://cloud.example.com", APIKey: "production"}); err != nil { + t.Fatal(err) + } + if err := development.WriteConfig(auth.Config{Host: "http://localhost:3000", APIKey: "development"}); err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + version string + wantKey string + }{ + {version: "dev", wantKey: "development"}, + {version: "v1.0.0", wantKey: "production"}, + } { + app := NewApp(tc.version, strings.NewReader(""), &bytes.Buffer{}, &bytes.Buffer{}) + config, err := app.requireConfig() + if err != nil { + t.Fatal(err) + } + if config.APIKey != tc.wantKey { + t.Fatalf("NewApp(%q) API key = %q, want %q", tc.version, config.APIKey, tc.wantKey) + } + } +} + func TestAuthDeviceExchangeCreatesAPIKeyAndWhoamiUsesIt(t *testing.T) { - configHome(t) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "config")) polls := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { @@ -68,7 +98,7 @@ func TestAuthDeviceExchangeCreatesAPIKeyAndWhoamiUsesIt(t *testing.T) { if err := execute(app, "auth", "login", "--host", server.URL); err != nil { t.Fatal(err) } - cfg, _ := auth.ReadConfig() + cfg, _ := auth.NewConfigStore("test").ReadConfig() if polls != 1 || cfg == nil || cfg.APIKey != "new-secret" || cfg.KeyID != "key-id" { t.Fatalf("polls=%d config=%#v", polls, cfg) } @@ -104,8 +134,6 @@ func TestInitRecommendsLinkInHumanAndJSON(t *testing.T) { } func TestLinkByIDsFetchesConfigurationAndSupportsPublicGitHub(t *testing.T) { - configHome(t) - root := "cmd/api" var paths []string s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { paths = append(paths, r.URL.Path) @@ -117,7 +145,7 @@ func TestLinkByIDsFetchesConfigurationAndSupportsPublicGitHub(t *testing.T) { case "/api/v1/projects/p/environments/e/services": w.Write([]byte(`{"services":[{"id":"s","name":"web","source":{"type":"github","repository":"https://github.com/acme/public","branch":"main","rootDir":"cmd/api"}}]}`)) case "/api/v1/projects/p/environments/e/services/s/configuration": - w.Write([]byte(`{"current":{"replicas":2,"hostname":null,"ports":[],"healthCheck":null,"startCommand":null},"management":{"patchable":true,"blockers":[]}}`)) + w.Write([]byte(`{"current":{"replicas":2,"placement":{"mode":"manual"},"placements":[{"serverId":"server-a","count":1},{"serverId":"server-b","count":1}],"hostname":null,"ports":[],"healthCheck":null,"startCommand":null},"management":{"patchable":true,"blockers":[]}}`)) default: t.Errorf("path=%s", r.URL.Path) } @@ -133,15 +161,43 @@ func TestLinkByIDsFetchesConfigurationAndSupportsPublicGitHub(t *testing.T) { if err != nil { t.Fatal(err) } - if loaded.Manifest.Service.Source.Repository != "https://github.com/acme/public" || loaded.Manifest.Service.Source.RootDir == nil || *loaded.Manifest.Service.Source.RootDir != root || loaded.Manifest.Service.Replicas != 2 { + if loaded.Manifest.Service.Source.Repository != "https://github.com/acme/public" || loaded.Manifest.Service.Source.RootDir == nil || *loaded.Manifest.Service.Source.RootDir != "cmd/api" || loaded.Manifest.Service.Replicas != 2 { t.Fatalf("manifest=%#v", loaded.Manifest) } + if loaded.Manifest.Service.Placement == nil || loaded.Manifest.Service.Placement.Mode != "manual" || len(loaded.Manifest.Service.Placement.Servers) != 2 || loaded.Manifest.Service.Placement.Servers[1].ServerID != "server-b" { + t.Fatalf("placement=%#v", loaded.Manifest.Service.Placement) + } want := []string{"/api/v1/projects", "/api/v1/projects/p/environments", "/api/v1/projects/p/environments/e/services", "/api/v1/projects/p/environments/e/services/s/configuration"} if !reflect.DeepEqual(paths, want) { t.Fatalf("paths=%v", paths) } } +func TestLinkRejectsManualServiceWithoutPlacements(t *testing.T) { + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/projects": + w.Write([]byte(`{"projects":[{"id":"p","name":"App","slug":"app"}]}`)) + case "/api/v1/projects/p/environments": + w.Write([]byte(`{"environments":[{"id":"e","name":"prod"}]}`)) + case "/api/v1/projects/p/environments/e/services": + w.Write([]byte(`{"services":[{"id":"s","name":"web","source":{"type":"image","image":"nginx"}}]}`)) + case "/api/v1/projects/p/environments/e/services/s/configuration": + w.Write([]byte(`{"current":{"replicas":0,"placement":{"mode":"manual"},"placements":[],"hostname":null,"ports":[],"healthCheck":null,"startCommand":null},"management":{"patchable":true,"blockers":[]}}`)) + default: + t.Errorf("path=%s", r.URL.Path) + } + })) + defer s.Close() + writeConfig(t, s.URL) + d := t.TempDir() + app, _ := testApp(t, d, s.Client()) + err := execute(app, "link", "--project", "p", "--environment", "e", "--service", "s") + if err == nil || !strings.Contains(err.Error(), "configure at least one server placement") { + t.Fatalf("error = %v", err) + } +} + func TestApplyExactNestedPatchForSources(t *testing.T) { for _, tc := range []struct{ name, source, sourceType string }{ {"image", "{type: image, image: nginx:1.27}", "image"}, @@ -149,7 +205,6 @@ func TestApplyExactNestedPatchForSources(t *testing.T) { {"github_clear_root", "{type: github, repository: https://github.com/acme/repo, branch: main}", "github"}, } { t.Run(tc.name, func(t *testing.T) { - configHome(t) d := t.TempDir() writeManifest(t, d, strings.Replace(imageManifest, "{type: image, image: nginx:1.27}", tc.source, 1)) var method, path string @@ -169,7 +224,8 @@ func TestApplyExactNestedPatchForSources(t *testing.T) { t.Fatalf("%s %s", method, path) } source := body["source"].(map[string]any) - if source["type"] != tc.sourceType || body["replicas"] != float64(2) || len(body) != 6 { + placement := body["placement"].(map[string]any) + if source["type"] != tc.sourceType || placement["mode"] != "automatic" || placement["replicas"] != float64(2) || len(body) != 6 { t.Fatalf("body=%#v", body) } if tc.name == "github_clear_root" { @@ -182,6 +238,42 @@ func TestApplyExactNestedPatchForSources(t *testing.T) { } } +func TestApplyPlacementPayloads(t *testing.T) { + for _, tc := range []struct { + name, yaml string + want map[string]any + }{ + {"automatic", "", map[string]any{"mode": "automatic", "replicas": float64(2)}}, + {"manual", " placement:\n mode: manual\n servers:\n - {serverId: server-a, count: 2}\n", map[string]any{"mode": "manual", "placements": []any{map[string]any{"serverId": "server-a", "count": float64(2)}}}}, + } { + t.Run(tc.name, func(t *testing.T) { + d := t.TempDir() + manifestYAML := imageManifest + if tc.yaml != "" { + manifestYAML = strings.Replace(manifestYAML, " placement: {mode: automatic}\n", tc.yaml, 1) + } + writeManifest(t, d, manifestYAML) + var body map[string]any + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewDecoder(r.Body).Decode(&body) + w.Write([]byte(`{"action":"updated"}`)) + })) + defer s.Close() + writeConfig(t, s.URL) + app, _ := testApp(t, d, s.Client()) + if err := execute(app, "apply"); err != nil { + t.Fatal(err) + } + if _, exists := body["replicas"]; exists { + t.Fatalf("top-level replicas present: %#v", body) + } + if !reflect.DeepEqual(body["placement"], tc.want) { + t.Fatalf("placement=%#v want=%#v", body["placement"], tc.want) + } + }) + } +} + func TestCollectionRequestsFollowPagination(t *testing.T) { queries := map[string][]string{} s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -244,7 +336,6 @@ func TestCollectionRequestsFollowPagination(t *testing.T) { } func TestProjectsCommandListsProjects(t *testing.T) { - configHome(t) s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/v1/projects" || r.URL.Query().Get("limit") != "100" { t.Errorf("request = %s?%s", r.URL.Path, r.URL.RawQuery) @@ -262,19 +353,11 @@ func TestProjectsCommandListsProjects(t *testing.T) { if err := execute(app, "projects"); err != nil { t.Fatal(err) } - for _, value := range []string{"Projects", "p1", "App One", "app-one", "p2", "App Two", "app-two"} { - if !strings.Contains(out.String(), value) { - t.Fatalf("output missing %q: %s", value, out.String()) - } - } + assertHumanOutput(t, out.String(), "Projects", "p1", "App One", "app-one", "p2", "App Two", "app-two") } func TestProjectsCommandMachineOutput(t *testing.T) { - configHome(t) - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"projects":[{"id":"p1","name":"App","slug":"app"}]}`)) - })) - defer s.Close() + s := responseServer(t, `{"projects":[{"id":"p1","name":"App","slug":"app"}]}`) writeConfig(t, s.URL) for _, mode := range []string{"--agent", "--json"} { @@ -315,8 +398,22 @@ func TestProjectsCommandHelpAndArguments(t *testing.T) { } } +func TestCompletionAgentHelpArguments(t *testing.T) { + app, out := testApp(t, t.TempDir(), nil) + if err := execute(app, "completion", "--help", "--agent"); err != nil { + t.Fatal(err) + } + var help agentHelpInfo + if err := json.Unmarshal(out.Bytes(), &help); err != nil { + t.Fatal(err) + } + want := []agentArg{{Name: "shell", Required: true, Choices: []string{"bash", "zsh", "fish", "powershell"}}} + if !reflect.DeepEqual(help.Args, want) { + t.Fatalf("arguments = %#v, want %#v", help.Args, want) + } +} + func TestMissingIDsFailLocally(t *testing.T) { - configHome(t) writeConfig(t, "http://unused") for _, command := range []string{"apply", "deploy", "status", "logs"} { t.Run(command, func(t *testing.T) { @@ -345,7 +442,6 @@ func TestDeploySourceNeutralAndMismatch(t *testing.T) { {"mismatch", `{type: image, image: nginx:1.27}`, `{"type":"image","image":"nginx:latest"}`, true}, } { t.Run(tc.name, func(t *testing.T) { - configHome(t) d := t.TempDir() writeManifest(t, d, strings.Replace(imageManifest, `{type: image, image: nginx:1.27}`, tc.local, 1)) posts := 0 @@ -388,7 +484,6 @@ func TestStatusAndResourceRoutesAndOutput(t *testing.T) { } for _, tc := range commands { t.Run(strings.Join(tc.args, "_"), func(t *testing.T) { - configHome(t) var gotPath, gotQuery string s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotPath, gotQuery = r.URL.Path, r.URL.RawQuery @@ -434,11 +529,7 @@ func TestBuildsHumanOutputIsFormatted(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - configHome(t) - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(tc.response)) - })) - defer s.Close() + s := responseServer(t, tc.response) writeConfig(t, s.URL) app, out := testApp(t, t.TempDir(), s.Client()) @@ -446,24 +537,13 @@ func TestBuildsHumanOutputIsFormatted(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(out.String(), "{") { - t.Fatalf("human output contains raw JSON:\n%s", out.String()) - } - for _, want := range tc.want { - if !strings.Contains(out.String(), want) { - t.Fatalf("output missing %q:\n%s", want, out.String()) - } - } + assertHumanOutput(t, out.String(), tc.want...) }) } } func TestConfigurationHumanOutputIsFormatted(t *testing.T) { - configHome(t) - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"current":{"source":{"type":"github","repository":"https://github.com/acme/app","branch":"main","rootDir":"cmd/api"},"hostname":"api.example.com","stateful":true,"replicas":2,"placements":[{"serverId":"server-1","serverName":"ubuntu-2","count":2}],"healthCheck":{"cmd":"curl localhost:3000","interval":10,"timeout":5,"retries":3,"startPeriod":30},"startCommand":"npm start","resources":{"cpuCores":2,"memoryMb":512},"ports":[{"containerPort":3000,"public":true,"domain":"api.example.com","protocol":"http","externalPort":null}],"volumes":[{"name":"data","containerPath":"/data"}],"serverless":{"enabled":false,"sleepAfterSeconds":300,"wakeTimeoutSeconds":30},"schedules":{"deployment":null,"backup":{"enabled":false,"schedule":null}}},"active":null,"activeRevisionId":"0400075c-69aa-46c2-bccc-fc172b8c6b28","activeDeploymentId":"2c917d90-4bc1-4274-b3bf-34fed009fc12","hasPendingChanges":true,"changes":[{"field":"replicas","from":"active revision","to":"current configuration"}],"management":{"patchable":true,"blockers":[]}}`)) - })) - defer s.Close() + s := responseServer(t, `{"current":{"source":{"type":"github","repository":"https://github.com/acme/app","branch":"main","rootDir":"cmd/api"},"hostname":"api.example.com","stateful":true,"replicas":2,"placements":[{"serverId":"server-1","serverName":"ubuntu-2","count":2}],"healthCheck":{"cmd":"curl localhost:3000","interval":10,"timeout":5,"retries":3,"startPeriod":30},"startCommand":"npm start","resources":{"cpuCores":2,"memoryMb":512},"ports":[{"containerPort":3000,"public":true,"domain":"api.example.com","protocol":"http","externalPort":null}],"volumes":[{"name":"data","containerPath":"/data"}],"serverless":{"enabled":false,"sleepAfterSeconds":300,"wakeTimeoutSeconds":30},"schedules":{"deployment":null,"backup":{"enabled":false,"schedule":null}}},"active":null,"activeRevisionId":"0400075c-69aa-46c2-bccc-fc172b8c6b28","activeDeploymentId":"2c917d90-4bc1-4274-b3bf-34fed009fc12","hasPendingChanges":true,"changes":[{"field":"replicas","from":"active revision","to":"current configuration"}],"management":{"patchable":true,"blockers":[]}}`) writeConfig(t, s.URL) app, out := testApp(t, t.TempDir(), s.Client()) @@ -471,22 +551,11 @@ func TestConfigurationHumanOutputIsFormatted(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(out.String(), "{") { - t.Fatalf("human output contains raw JSON:\n%s", out.String()) - } - for _, want := range []string{"Configuration", "https://github.com/acme/app @ main (cmd/api)", "api.example.com", "ubuntu-2: 2 replica(s)", "3000/http public", "data: /data", "curl localhost:3000", "Deployment", "0400075c...6b28", "replicas: active revision -> current configuration", "Management", "Patchable", "yes"} { - if !strings.Contains(out.String(), want) { - t.Fatalf("output missing %q:\n%s", want, out.String()) - } - } + assertHumanOutput(t, out.String(), "Configuration", "https://github.com/acme/app @ main (cmd/api)", "api.example.com", "ubuntu-2: 2 replica(s)", "3000/http public", "data: /data", "curl localhost:3000", "Deployment", "0400075c...6b28", "replicas: active revision -> current configuration", "Management", "Patchable", "yes") } func TestMetricsHumanOutputIsFormatted(t *testing.T) { - configHome(t) - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"provider":"enabled","metrics":{"range":"1h","windowStart":"2026-07-21T09:00:00Z","windowEnd":"2026-07-21T10:00:00Z","totalRequests":1000000,"totalIngressBytes":460,"totalEgressBytes":2048,"buckets":[{"timestamp":"2026-07-21T10:00:00Z","totalRequests":1000000,"cpuUsagePercent":42.5,"memoryUsagePercent":50,"memoryUsedBytes":1048576,"p50ResponseTimeMs":12,"p95ResponseTimeMs":45,"ingressBytesPerSecond":512,"egressBytesPerSecond":1024}]}}`)) - })) - defer s.Close() + s := responseServer(t, `{"provider":"enabled","metrics":{"range":"1h","windowStart":"2026-07-21T09:00:00Z","windowEnd":"2026-07-21T10:00:00Z","totalRequests":1000000,"totalIngressBytes":460,"totalEgressBytes":2048,"buckets":[{"timestamp":"2026-07-21T10:00:00Z","totalRequests":1000000,"cpuUsagePercent":42.5,"memoryUsagePercent":50,"memoryUsedBytes":1048576,"p50ResponseTimeMs":12,"p95ResponseTimeMs":45,"ingressBytesPerSecond":512,"egressBytesPerSecond":1024}]}}`) writeConfig(t, s.URL) app, out := testApp(t, t.TempDir(), s.Client()) @@ -494,25 +563,14 @@ func TestMetricsHumanOutputIsFormatted(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(out.String(), "{") { - t.Fatalf("human output contains raw JSON:\n%s", out.String()) - } - for _, want := range []string{"Metrics", "1h", "Requests", "1000000", "2.0 KiB", "Latest sample", "42.5%", "50% (1.0 MiB)", "12 ms", "45 ms", "512 B/s", "1024 B/s"} { - if !strings.Contains(out.String(), want) { - t.Fatalf("output missing %q:\n%s", want, out.String()) - } - } + assertHumanOutput(t, out.String(), "Metrics", "1h", "Requests", "1000000", "2.0 KiB", "Latest sample", "42.5%", "50% (1.0 MiB)", "12 ms", "45 ms", "512 B/s", "1024 B/s") if strings.Contains(out.String(), "1e+06") { t.Fatalf("request count uses scientific notation:\n%s", out.String()) } } func TestMetricsHumanOutputShowsDisabledProvider(t *testing.T) { - configHome(t) - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"provider":"disabled","metrics":null}`)) - })) - defer s.Close() + s := responseServer(t, `{"provider":"disabled","metrics":null}`) writeConfig(t, s.URL) app, out := testApp(t, t.TempDir(), s.Client()) @@ -520,17 +578,11 @@ func TestMetricsHumanOutputShowsDisabledProvider(t *testing.T) { if err != nil { t.Fatal(err) } - if !strings.Contains(out.String(), "Status") || !strings.Contains(out.String(), "disabled") || strings.Contains(out.String(), "{") { - t.Fatalf("output = %s", out.String()) - } + assertHumanOutput(t, out.String(), "Status", "disabled") } func TestRevisionsHumanOutputIsFormatted(t *testing.T) { - configHome(t) - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(`{"nextCursor":"next-page","revisions":[{"id":"0400075c-69aa-46c2-bccc-fc172b8c6b28","createdAt":"2026-07-21T11:04:58.816Z","actor":{"name":"Arjun Komath","type":"user"},"comparison":{"kind":"changes","changes":[{"field":"Image","from":"app:v1","to":"app:v2"}]},"rollout":{"id":"2c917d90-4bc1-4274-b3bf-34fed009fc12","status":"in_progress"}},{"id":"3f9293e9-d4d8-4b72-8ae4-94b3737ab056","createdAt":"2026-07-21T02:16:12.820Z","actor":{"type":"github","login":"octocat"},"comparison":{"kind":"initial"},"rollout":null}]}`)) - })) - defer s.Close() + s := responseServer(t, `{"nextCursor":"next-page","revisions":[{"id":"0400075c-69aa-46c2-bccc-fc172b8c6b28","createdAt":"2026-07-21T11:04:58.816Z","actor":{"name":"Arjun Komath","type":"user"},"comparison":{"kind":"changes","changes":[{"field":"Image","from":"app:v1","to":"app:v2"}]},"rollout":{"id":"2c917d90-4bc1-4274-b3bf-34fed009fc12","status":"in_progress"}},{"id":"3f9293e9-d4d8-4b72-8ae4-94b3737ab056","createdAt":"2026-07-21T02:16:12.820Z","actor":{"type":"github","login":"octocat"},"comparison":{"kind":"initial"},"rollout":null}]}`) writeConfig(t, s.URL) app, out := testApp(t, t.TempDir(), s.Client()) @@ -538,18 +590,10 @@ func TestRevisionsHumanOutputIsFormatted(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(out.String(), "{") { - t.Fatalf("human output contains raw JSON:\n%s", out.String()) - } - for _, want := range []string{"Revisions (2)", "0400075c...6b28", "Arjun Komath", "in progress", "Image: app:v1 -> app:v2", "3f9293e9...b056", "@octocat", "initial revision", "next-page"} { - if !strings.Contains(out.String(), want) { - t.Fatalf("output missing %q:\n%s", want, out.String()) - } - } + assertHumanOutput(t, out.String(), "Revisions (2)", "0400075c...6b28", "Arjun Komath", "in progress", "Image: app:v1 -> app:v2", "3f9293e9...b056", "@octocat", "initial revision", "next-page") } func TestRolloutHumanOutputIsFormatted(t *testing.T) { - configHome(t) s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/api/v1/projects/p/environments/e/services/s/rollouts": @@ -578,19 +622,11 @@ func TestRolloutHumanOutputIsFormatted(t *testing.T) { if err := execute(app, args...); err != nil { t.Fatal(err) } - if strings.Contains(out.String(), "{") { - t.Fatalf("human output contains raw JSON:\n%s", out.String()) - } - for _, want := range tc.want { - if !strings.Contains(out.String(), want) { - t.Fatalf("output missing %q:\n%s", want, out.String()) - } - } + assertHumanOutput(t, out.String(), tc.want...) } } func TestLogsQueryCursorLongPollAndCancellation(t *testing.T) { - configHome(t) requests := 0 ctx, cancel := context.WithCancel(context.Background()) var queries []string @@ -621,7 +657,6 @@ func TestLogsQueryCursorLongPollAndCancellation(t *testing.T) { } func TestLogsDrainAvailablePagesWithoutSleeping(t *testing.T) { - configHome(t) requests := 0 ctx, cancel := context.WithCancel(context.Background()) s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -661,6 +696,27 @@ func TestLogsDrainAvailablePagesWithoutSleeping(t *testing.T) { } } +func assertHumanOutput(t *testing.T, got string, want ...string) { + t.Helper() + if strings.Contains(got, "{") { + t.Fatalf("human output contains raw JSON:\n%s", got) + } + for _, value := range want { + if !strings.Contains(got, value) { + t.Fatalf("output missing %q:\n%s", value, got) + } + } +} + +func responseServer(t *testing.T, body string) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + func testApp(t *testing.T, cwd string, client *http.Client) (*App, *bytes.Buffer) { t.Helper() var out bytes.Buffer @@ -682,13 +738,10 @@ func execute(a *App, args ...string) error { return c.Execute() } -func configHome(t *testing.T) { - t.Helper() - t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "config")) -} func writeConfig(t *testing.T, host string) { t.Helper() - if err := auth.WriteConfig(auth.Config{Host: host, APIKey: "secret"}); err != nil { + t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "config")) + if err := auth.NewConfigStore("test").WriteConfig(auth.Config{Host: host, APIKey: "secret"}); err != nil { t.Fatal(err) } } diff --git a/cli/internal/manifest/manifest.go b/cli/internal/manifest/manifest.go index 8891aa94..76136a77 100644 --- a/cli/internal/manifest/manifest.go +++ b/cli/internal/manifest/manifest.go @@ -36,10 +36,19 @@ type Service struct { Hostname *string `json:"hostname" yaml:"hostname"` Ports []Port `json:"ports" yaml:"ports"` Replicas int `json:"replicas" yaml:"replicas"` + Placement *Placement `json:"placement,omitempty" yaml:"placement,omitempty"` HealthCheck *HealthCheck `json:"healthCheck" yaml:"healthCheck"` StartCommand *string `json:"startCommand" yaml:"startCommand"` Resources *Resources `json:"resources,omitempty" yaml:"resources,omitempty"` } +type Placement struct { + Mode string `json:"mode" yaml:"mode"` + Servers []PlacementServer `json:"servers,omitempty" yaml:"servers,omitempty"` +} +type PlacementServer struct { + ServerID string `json:"serverId" yaml:"serverId"` + Count int `json:"count" yaml:"count"` +} type Source struct { Type string `json:"type" yaml:"type"` Image string `json:"image,omitempty" yaml:"image,omitempty"` @@ -140,6 +149,12 @@ func ApplyDefaults(m *Manifest) { if m.Service.Replicas == 0 { m.Service.Replicas = 1 } + if p := m.Service.Placement; p != nil { + p.Mode = strings.ToLower(strings.TrimSpace(p.Mode)) + for i := range p.Servers { + p.Servers[i].ServerID = strings.TrimSpace(p.Servers[i].ServerID) + } + } if h := m.Service.HealthCheck; h != nil { h.Cmd = strings.TrimSpace(h.Cmd) if h.Interval == 0 { @@ -213,6 +228,42 @@ func Validate(m Manifest) error { if m.Service.Replicas < 1 || m.Service.Replicas > 10 { return errors.New("service.replicas must be between 1 and 10") } + if m.Service.Placement == nil { + return errors.New("service.placement is required") + } + if p := m.Service.Placement; p != nil { + switch p.Mode { + case "automatic": + if p.Servers != nil { + return errors.New("service.placement.servers cannot be set for automatic placement") + } + case "manual": + total := 0 + seen := make(map[string]struct{}, len(p.Servers)) + for i, server := range p.Servers { + serverID := strings.TrimSpace(server.ServerID) + if serverID == "" { + return fmt.Errorf("service.placement.servers[%d].serverId cannot be blank", i) + } + if _, exists := seen[serverID]; exists { + return fmt.Errorf("service.placement.servers[%d].serverId must be unique", i) + } + seen[serverID] = struct{}{} + if server.Count < 1 { + return fmt.Errorf("service.placement.servers[%d].count must be positive", i) + } + total += server.Count + } + if total < 1 || total > 10 { + return errors.New("service.placement manual total must be between 1 and 10") + } + if total != m.Service.Replicas { + return errors.New("service.placement manual total must equal service.replicas") + } + default: + return errors.New("service.placement.mode must be automatic or manual") + } + } seenPorts := make(map[int]struct{}, len(m.Service.Ports)) for i, p := range m.Service.Ports { if p.ContainerPort < 1 || p.ContainerPort > 65535 { diff --git a/cli/internal/manifest/manifest_test.go b/cli/internal/manifest/manifest_test.go index 39eaf9a3..83e3a025 100644 --- a/cli/internal/manifest/manifest_test.go +++ b/cli/internal/manifest/manifest_test.go @@ -6,7 +6,7 @@ import ( ) func base() Manifest { - return Manifest{APIVersion: "v1", Project: Project{ID: "p", Slug: "app"}, Environment: Environment{ID: "e", Name: "prod"}, Service: Service{ID: "s", Name: "web", Source: Source{Type: "image", Image: "nginx"}, Replicas: 1}} + return Manifest{APIVersion: "v1", Project: Project{ID: "p", Slug: "app"}, Environment: Environment{ID: "e", Name: "prod"}, Service: Service{ID: "s", Name: "web", Source: Source{Type: "image", Image: "nginx"}, Replicas: 1, Placement: &Placement{Mode: "automatic"}}} } func TestDefaultsAndRoundTrip(t *testing.T) { m := base() @@ -20,6 +20,62 @@ func TestDefaultsAndRoundTrip(t *testing.T) { t.Fatalf("got=%#v err=%v", got, e) } } + +func TestPlacementRoundTripAndValidation(t *testing.T) { + m := base() + m.Service.Replicas = 3 + m.Service.Placement = &Placement{Mode: " manual ", Servers: []PlacementServer{{ServerID: " server-a ", Count: 2}, {ServerID: "server-b", Count: 1}}} + b, err := Marshal(m) + if err != nil { + t.Fatal(err) + } + got, err := Parse(b) + if err != nil { + t.Fatal(err) + } + if got.Service.Placement == nil || got.Service.Placement.Mode != "manual" || got.Service.Placement.Servers[0].ServerID != "server-a" { + t.Fatalf("placement=%#v", got.Service.Placement) + } + + tests := []struct { + name string + placement *Placement + replicas int + }{ + {"invalid mode", &Placement{Mode: "random"}, 1}, + {"automatic servers", &Placement{Mode: "automatic", Servers: []PlacementServer{}}, 1}, + {"blank server", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: " ", Count: 1}}}, 1}, + {"duplicate server", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: "a", Count: 1}, {ServerID: "a", Count: 1}}}, 2}, + {"nonpositive count", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: "a", Count: 0}}}, 1}, + {"total exceeds limit", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: "a", Count: 11}}}, 10}, + {"total differs from replicas", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: "a", Count: 1}}}, 2}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := base() + m.Service.Replicas = tc.replicas + m.Service.Placement = tc.placement + ApplyDefaults(&m) + if err := Validate(m); err == nil { + t.Fatal("invalid placement accepted") + } + }) + } +} + +func TestPlacementIsRequired(t *testing.T) { + _, err := Parse([]byte(`apiVersion: v1 +project: {slug: app} +environment: {name: prod} +service: + name: web + source: {type: image, image: nginx} + replicas: 2 +`)) + if err == nil || !strings.Contains(err.Error(), "service.placement is required") { + t.Fatalf("error = %v", err) + } +} func TestGitHubCanonical(t *testing.T) { m := base() root := `packages\web` diff --git a/cli/internal/output/output.go b/cli/internal/output/output.go index 2a49d93d..b961b31d 100644 --- a/cli/internal/output/output.go +++ b/cli/internal/output/output.go @@ -29,11 +29,7 @@ func OK(w io.Writer, data any, summary string) error { } func Error(w io.Writer, err error) error { - message := "" - if err != nil { - message = err.Error() - } - return JSON(w, ErrorEnvelope{OK: false, Error: message}) + return JSON(w, ErrorEnvelope{OK: false, Error: err.Error()}) } func Section(w io.Writer, title string) { diff --git a/deployment/README.md b/deployment/README.md index 0a5720c5..7ffdcfd8 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -80,10 +80,16 @@ After signing in as an admin, configure **Edge Domain** under public edge traffic, including HTTP/HTTPS custom domains and direct TCP/UDP connection strings. -Configure this hostname in your DNS provider to resolve to every proxy server's -public IPv4 address. You can use direct `A` records or your own DNS routing, -load-balancing, or GeoDNS policy. Custom HTTP/HTTPS subdomains can use a `CNAME` -to the edge domain; apex domains can use `ALIAS` or `ANAME` where supported. +A stable external load balancer with active health checks is the ideal production +solution for proxy failure. Configure each proxy public IPv4 address as an origin, +then point the edge hostname to the load balancer's stable address. A direct `A` +record to one proxy has no ingress failover. Multiple proxy `A` records provide +best-effort DNS distribution, but cached answers may continue sending clients to +an offline proxy. + +Custom HTTP/HTTPS subdomains can use a `CNAME` to the edge domain; apex domains +can use `ALIAS` or `ANAME` where supported. Health-aware GeoDNS is an alternative, +but its failover time remains subject to DNS and client caching. Techulus Cloud only displays the required DNS configuration. It does not create or update DNS records. @@ -94,8 +100,7 @@ Configure **Automatic Subdomain Domain** under **Settings → Infrastructure** t offer generated domains in service Networking settings. Enter only the base domain, such as `apps.example.com`, without `*.` or a protocol. -Create wildcard DNS records for `*.apps.example.com` that resolve to every proxy -server public IPv4 address. +Create a wildcard `CNAME` for `*.apps.example.com` that points to the edge domain. ### Web Replicas diff --git a/deployment/updater/Dockerfile b/deployment/updater/Dockerfile index f463f92f..ccadda10 100644 --- a/deployment/updater/Dockerfile +++ b/deployment/updater/Dockerfile @@ -14,9 +14,7 @@ COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker COPY --from=docker-cli /usr/local/libexec/docker/cli-plugins/docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose RUN pg_dump --version | grep -Eq 'PostgreSQL\) 18\.' \ - && docker --version \ - && docker compose version \ - && curl --version + && docker compose version WORKDIR /app COPY --from=builder /out/control-plane-updater ./control-plane-updater diff --git a/deployment/updater/main.go b/deployment/updater/main.go index 14657eb4..ce9d0c2b 100644 --- a/deployment/updater/main.go +++ b/deployment/updater/main.go @@ -33,7 +33,6 @@ type server struct { token string rawBaseURL string healthURL string - statusFile string mu sync.Mutex status updaterStatus @@ -42,13 +41,11 @@ type server struct { var versionPattern = regexp.MustCompile(`^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$`) func main() { - deployDir := getenv("DEPLOY_DIR", "/opt/techulus-cloud") s := &server{ - deployDir: deployDir, + deployDir: getenv("DEPLOY_DIR", "/opt/techulus-cloud"), token: os.Getenv("CONTROL_PLANE_UPDATER_TOKEN"), rawBaseURL: getenv("RAW_BASE_URL", "https://raw.githubusercontent.com/techulus/cloud"), healthURL: getenv("WEB_HEALTH_URL", "http://web:3000/api/health"), - statusFile: filepath.Join(deployDir, "updater-status.json"), } s.status = s.readStatus() @@ -74,7 +71,7 @@ func getenv(key, fallback string) string { } func (s *server) readStatus() updaterStatus { - data, err := os.ReadFile(s.statusFile) + data, err := os.ReadFile(filepath.Join(s.deployDir, "updater-status.json")) if err != nil { return updaterStatus{Status: "idle", Logs: []string{}} } @@ -99,7 +96,7 @@ func (s *server) persistStatusLocked() { log.Printf("failed to marshal updater status: %v", err) return } - if err := os.WriteFile(s.statusFile, data, 0o600); err != nil { + if err := os.WriteFile(filepath.Join(s.deployDir, "updater-status.json"), data, 0o600); err != nil { log.Printf("failed to persist updater status: %v", err) } } @@ -456,10 +453,7 @@ func (s *server) run(name string, args []string, displayArgs []string) error { go s.pipeLogs(&wg, stderr) wg.Wait() - if err := cmd.Wait(); err != nil { - return err - } - return nil + return cmd.Wait() } func (s *server) pipeLogs(wg *sync.WaitGroup, reader io.Reader) { @@ -480,15 +474,14 @@ func (s *server) pipeLogs(wg *sync.WaitGroup, reader io.Reader) { func (s *server) runOutput(name string, args []string) (string, error) { cmd := exec.Command(name, args...) cmd.Dir = s.deployDir - var stdout bytes.Buffer var stderr bytes.Buffer - cmd.Stdout = &stdout cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { + stdout, err := cmd.Output() + if err != nil { if stderr.Len() > 0 { return "", errors.New(strings.TrimSpace(stderr.String())) } return "", err } - return stdout.String(), nil + return string(stdout), nil } diff --git a/docs/agents/updating.mdx b/docs/agents/updating.mdx index 7fcdc682..394dee88 100644 --- a/docs/agents/updating.mdx +++ b/docs/agents/updating.mdx @@ -1,39 +1,35 @@ --- title: "Updating" -description: "Update agents to the latest version." +description: "Update agents from the control plane." --- -## Automated Update +## Update an agent -Run the update script on the server: +When an update is available, open the server in the control plane and click **Update**. Confirm the target version, then click **Queue upgrade**. -```bash -curl -sSL https://your-control-plane.com/update.sh | sudo bash -``` - -The script: +The control plane sends a work item over the authenticated agent channel. The agent: -1. Downloads the latest agent binary from GitHub releases. +1. Downloads the target version of the agent binary from GitHub releases. 2. Verifies the SHA256 checksum. 3. Replaces the existing binary. -4. Restarts the systemd service. +4. Restarts itself after installation. No configuration or dependencies are changed during an update. -## Manual Update +## Manual update Download the binary for your architecture: ```bash # For x86_64 curl -fsSL https://github.com/techulus/cloud/releases/latest/download/agent-linux-amd64 \ - -o /usr/local/bin/agent -chmod +x /usr/local/bin/agent + -o /usr/local/bin/techulus-agent +chmod +x /usr/local/bin/techulus-agent # For arm64 curl -fsSL https://github.com/techulus/cloud/releases/latest/download/agent-linux-arm64 \ - -o /usr/local/bin/agent -chmod +x /usr/local/bin/agent + -o /usr/local/bin/techulus-agent +chmod +x /usr/local/bin/techulus-agent ``` Restart the service: diff --git a/docs/api/public-api.mdx b/docs/api/public-api.mdx index c794565e..5fb9c8ae 100644 --- a/docs/api/public-api.mdx +++ b/docs/api/public-api.mdx @@ -156,7 +156,10 @@ Configuration and revision responses never include secret names, values, or ciph "domain": "cloud.example.com" } ], - "replicas": 1, + "placement": { + "mode": "automatic", + "replicas": 3 + }, "healthCheck": { "cmd": "curl --fail http://localhost:3000/health", "interval": 10, @@ -189,7 +192,26 @@ The API supports these source variants: GitHub repository URLs are canonical HTTPS `github.com` URLs. `rootDir` must stay inside the repository. Use `rootDir: null` to clear an existing build root. Source conversion and GitHub repository switching are not supported. -The API only manages stateless services with HTTP ports. Existing volumes, stateful mode, TCP or UDP ports, TLS passthrough, unsupported placement, or invalid resource limits return a conflict with an actionable code. `replicas` must match the existing server placement. +Use automatic placement for stateless services: + +```json +{ "mode": "automatic", "replicas": 3 } +``` + +Use manual placement to choose exact servers: + +```json +{ + "mode": "manual", + "placements": [ + { "serverId": "server-id", "count": 2 } + ] +} +``` + +Manual placement requires online servers with WireGuard configured. Serverless services require proxy servers. Automatic placement is not available for stateful, serverless, or volume-backed services. Submit replica changes through `placement`; the API rejects a top-level `replicas` field. + +The API only manages stateless services with HTTP ports. Existing volumes, stateful mode, TCP or UDP ports, TLS passthrough, or invalid resource limits return a conflict with an actionable code. ## Deployments and builds diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 0bc4d385..a7de10d8 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -210,8 +210,8 @@ Worker nodes do not run Traefik. Public HTTP services can be configured as serverless. Serverless scale-to-zero is proxy-local: deployments placed on proxy nodes may sleep after an idle period, -then wake on the next public request handled by that proxy. Deployments placed on -worker nodes stay always on and remain routable. +then wake on the next public request handled by that proxy. Serverless services +use manual placement on proxy nodes. Serverless uses the same declarative expected-state model as normal deployments, but proxy agents own the local lifecycle decision: @@ -220,8 +220,6 @@ deployments, but proxy agents own the local lifecycle decision: `runtimeDesiredState: "stopped"` and `observedPhase: "sleeping"`. - Expected state advertises proxy-hosted sleeping containers with `desiredState: "stopped"` so normal reconciliation does not restart them. -- Expected state advertises worker-hosted deployments with - `desiredState: "running"` even when the service is serverless-enabled. - The proxy gateway wakes local sleeping deployments from expected-state metadata and reports `wake_started` through the next status report. - The proxy agent reports local sleep with a `sleep` status transition after it @@ -248,18 +246,15 @@ sequenceDiagram Proxy Traefik routes point to the local wake gateway only on proxy nodes that host a local proxy deployment for that serverless service. Non-owner proxies do -not emit a public HTTP route for that service, even when always-on worker -upstreams exist. Worker-only serverless services use normal direct routes because -there is nothing local to wake. +not emit a public HTTP route for that service. Public DNS or external load balancers must therefore send serverless traffic only to proxy nodes that own a local proxy replica for that service. Cross-proxy wake coordination is intentionally out of scope. The wake gateway keeps the incoming request open while it starts local proxy -replicas, unless an always-on worker upstream is already ready. Queued requests -resume when one upstream is ready. If no upstream becomes ready and no local wake -is still in progress, the gateway returns a 503. +replicas. Queued requests resume when one upstream is ready. If no upstream +becomes ready and no local wake is still in progress, the gateway returns a 503. Sleep is driven by the proxy gateway's in-memory activity timer. The control plane does not scan request activity and does not enqueue sleep work. It accepts @@ -273,11 +268,13 @@ deployments. ### Multiple Proxy Nodes -The platform supports geographically distributed proxy nodes with proximity steering: +The platform supports multiple proxy nodes behind a shared edge hostname. A +stable external load balancer with active health checks is the ideal production +solution for proxy failure: -- Users point custom domains to a single GeoDNS-managed hostname. -- GeoDNS routes clients to the nearest healthy proxy. -- Health checks fail over automatically when a proxy becomes unavailable. +- Users point custom domains to one shared edge hostname. +- The load balancer sends new connections only to healthy proxy origins. +- Workload placement changes proxy routes without changing public DNS. - All proxies share the same TLS certificates from the control plane. Example: @@ -287,12 +284,16 @@ Proxy US: 1.2.3.4 Proxy EU: 5.6.7.8 Proxy SYD: 9.10.11.12 -GeoDNS: +External load balancer: example.com -> lb.techulus.cloud - -> route client to nearest proxy - -> fail over when a proxy is unhealthy + -> actively health-check proxy origins + -> stop new traffic to an unhealthy proxy ``` +Health-aware GeoDNS can provide proximity steering, but recovery remains subject +to resolver and client caching. Plain multiple A records are best-effort traffic +distribution, not reliable proxy failover. + ### Proximity-Aware Load Balancing Within a proxy node, traffic is distributed using weighted round-robin: diff --git a/docs/index.mdx b/docs/index.mdx index c42edd84..df09e725 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -25,7 +25,8 @@ All communication between nodes happens over an encrypted **WireGuard mesh netwo - **Persistent volumes** — attach named local volumes for stateful workloads with scheduled backups to S3-compatible storage. Replicated storage and HA failover are not yet supported. - **Service discovery** — containers resolve each other by name using `.internal` domains. - **Multi-environment** — run production, staging, and dev within the same project. -- **GeoDNS** — route users to the nearest proxy node with automatic failover. +- **External edge compatibility** — use a stable load balancer with active health + checks, or health-aware GeoDNS when DNS caching tradeoffs are acceptable. - **TCP/UDP proxy** — expose non-HTTP services like game servers or custom protocols. ## Next Steps diff --git a/docs/services/domains.mdx b/docs/services/domains.mdx index 197be2cc..9ec13ef4 100644 --- a/docs/services/domains.mdx +++ b/docs/services/domains.mdx @@ -7,7 +7,7 @@ description: "Bind custom domains with automatic HTTPS." Custom domains are bound to a specific service port. When you add a domain: -1. Point your domain's DNS to a proxy node's IP address (or your GeoDNS hostname). +1. Point your domain's DNS to your edge hostname. 2. Add the domain in the service port settings. 3. The platform automatically provisions a TLS certificate via Let's Encrypt. @@ -20,9 +20,9 @@ offer automatic domains in service networking settings. For example, with the setting configured as `apps.example.com`, a service whose private hostname is `project-api-production` can use `project-api-production.apps.example.com`. -Create a wildcard DNS record for `*.apps.example.com` that points to all proxy -servers. The setting contains only the base domain, without `*.` or a protocol. -Custom domains remain available alongside automatic domains. +Create a wildcard `CNAME` record for `*.apps.example.com` that points to your +edge hostname. The setting contains only the base domain, without `*.` or a +protocol. Custom domains remain available alongside automatic domains. ## TLS Certificates @@ -37,7 +37,14 @@ Certificates are: When using multiple proxy nodes for geographic distribution, all proxies share the same TLS certificates from the control plane. -Set up a GeoDNS hostname that routes clients to the nearest healthy proxy, then point your custom domains to that hostname. See the [Architecture](/architecture#multiple-proxy-nodes) page for details. +A stable external load balancer with active health checks is the ideal production +solution for proxy failure. Configure every proxy as an origin, point the edge +hostname to the load balancer, and point custom domains to the edge hostname. + +Health-aware GeoDNS is an alternative, but failover remains subject to DNS and +client caching. Plain multiple A records provide best-effort distribution and do +not guarantee that clients avoid an offline proxy. See the +[Architecture](/architecture#multiple-proxy-nodes) page for details. ## Protocols diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index 387d0115..cb1e3cb3 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -17,11 +17,6 @@ and stops the local container. The next public HTTP request wakes local proxy-hosted deployments and is held until an upstream is ready or the wake timeout is reached. -Worker-hosted deployments do not sleep. If a serverless-enabled service has only -worker replicas, it stays always on and routes directly. If a service has both -proxy and worker replicas, proxy replicas may sleep while worker replicas remain -routable. - For services with proxy-hosted serverless replicas, public traffic must be sent only to proxy nodes that host a local proxy replica for that service. Non-owner proxies do not emit a public HTTP route for the service, so DNS or the external @@ -36,15 +31,20 @@ Serverless settings are configured per service: | Wake timeout | `300s` | Maximum time the wake gateway waits for ready upstreams | A cold wake starts the sleeping local proxy replicas for that host. Held -requests resume when one upstream is ready. If a worker upstream is already -ready, the gateway can serve it immediately while local proxy replicas wake in -the background. +requests resume when one upstream is ready. Serverless services require at least one configured replica. ## Placement -Services use manual placement. Select the target servers and replica counts explicitly before deploying. +Stateless services support automatic and manual placement. Automatic placement +stores the desired replica count and distributes replicas across online, +configured nodes during rollout. Manual placement selects exact target servers +and replica counts. + +Serverless services currently require manual placement on proxy nodes. Automatic +serverless placement will remain unavailable until every healthy proxy can route +requests through the owning proxy's wake gateway. ## Server Pinning @@ -61,6 +61,6 @@ You can also manually lock any service to a specific server by setting the locke - Stateful services do not automatically fail over to another server. - Maximum 10 replicas per service. - Serverless scaling requires a public HTTP service domain. -- Sleep and wake are proxy-local. Worker replicas are intentionally always on. +- Sleep and wake are proxy-local; serverless replicas must be placed on proxy nodes. - Serverless traffic must be routed only to proxy nodes that own a local proxy replica for that service. - Proxy agents report sleep and wake transitions through normal status reports. diff --git a/proxy/README.md b/proxy/README.md index d9b4e631..8e216569 100644 --- a/proxy/README.md +++ b/proxy/README.md @@ -51,10 +51,20 @@ Access at `http://:8080/dashboard/` ## DNS Setup -Point your domain's DNS to the proxy node(s) public IP: - -- **Single proxy**: A record pointing to proxy IP -- **Multiple proxies**: Multiple A records for DNS round-robin, or use a load balancer +Point your domain at a stable edge address: + +- **Production**: use a stable external load balancer with active health checks, + and configure every proxy public IP as an origin +- **Single proxy**: an A record can point directly to the proxy, but there is no + ingress failover +- **Multiple A records**: provide best-effort distribution, not reliable failover; + clients may cache and continue using an offline proxy +- **Health-aware GeoDNS**: supported as an alternative, with failover still + subject to DNS and client caching + +A stable external load balancer with active health checks is the ideal production +solution for proxy failure. Workload rebalancing updates proxy routes; it does not +modify public DNS or remove failed proxies from external traffic. ## How It Works diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 2f3ce3da..1240d225 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import cronstrue from "cronstrue"; -import { and, desc, eq, inArray, isNotNull, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { ZodError, z } from "zod"; import { db } from "@/db"; @@ -32,7 +32,9 @@ import { requireDeveloperRole, verifyDeleteConfirmation } from "@/lib/auth"; import { deployServiceInternal } from "@/lib/deploy-service"; import { isObservedReady, + isRuntimeExpected, markDeploymentRemoved, + observedReadyPhases, runtimeExpectedStates, } from "@/lib/deployment-status"; import { validateDockerImageInternal } from "@/lib/docker-image"; @@ -40,7 +42,6 @@ import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import { restoreDrainingDeploymentsForRollback } from "@/lib/inngest/functions/rollout-utils"; import { allocatePort } from "@/lib/port-allocation"; -import { blocksProjectDeletion } from "@/lib/project-deletion"; import { containerPathSchema, githubRepoUrlSchema, @@ -111,7 +112,10 @@ export async function deleteProject( .from(deployments) .where(eq(deployments.serviceId, service.id)); - const hasActiveDeployments = activeDeployments.some(blocksProjectDeletion); + // "stopped" can be a sleeping, wakeable serverless deployment. + const hasActiveDeployments = activeDeployments.some( + ({ runtimeDesiredState }) => isRuntimeExpected(runtimeDesiredState), + ); if (hasActiveDeployments) { throw new Error( @@ -459,7 +463,7 @@ export async function deleteService( .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.observedPhase, ["running", "healthy"]), + inArray(deployments.observedPhase, observedReadyPhases), ), ) .then((r) => r[0]); @@ -920,6 +924,11 @@ export async function updateServiceServerlessSettings( } if (validated.enabled) { + if (service.placementMode === "automatic") { + throw new Error( + "Switch to manual placement before enabling serverless", + ); + } const publicHttpPorts = await tx .select({ id: servicePorts.id }) .from(servicePorts) @@ -984,9 +993,51 @@ export type ServiceConfigUpdate = { source?: { type: "image"; image: string }; healthCheck?: ServiceHealthCheckConfig | null; ports?: { add?: PortConfig[]; remove?: string[] }; - replicas?: { serverId: string; count: number }[]; + placement?: + | { mode: "automatic"; replicas: number } + | { + mode: "manual"; + placements: { serverId: string; count: number }[]; + }; }; +const placementInputSchema = z.discriminatedUnion("mode", [ + z.strictObject({ + mode: z.literal("automatic"), + replicas: z.number().int().min(1).max(10), + }), + z + .strictObject({ + mode: z.literal("manual"), + placements: z + .array( + z.strictObject({ + serverId: z.string().min(1), + count: z.number().int().min(1).max(10), + }), + ) + .min(1), + }) + .superRefine((value, context) => { + if ( + new Set(value.placements.map((item) => item.serverId)).size !== + value.placements.length + ) + context.addIssue({ + code: "custom", + message: "Server IDs must be unique", + path: ["placements"], + }); + const total = value.placements.reduce((sum, item) => sum + item.count, 0); + if (total > 10) + context.addIssue({ + code: "custom", + message: "Total replicas must be between 1 and 10", + path: ["placements"], + }); + }), +]); + export async function updateServiceConfig( serviceId: string, config: ServiceConfigUpdate, @@ -1106,33 +1157,72 @@ export async function updateServiceConfig( } } - if (config.replicas) { - const replicas = config.replicas; + if (config.placement) { + const placement = placementInputSchema.parse(config.placement); await db.transaction(async (tx) => { await tx.execute( sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, ); const [currentService] = await tx - .select({ serverlessEnabled: services.serverlessEnabled }) + .select({ + serverlessEnabled: services.serverlessEnabled, + stateful: services.stateful, + placementMode: services.placementMode, + }) .from(services) .where(eq(services.id, serviceId)) .limit(1); - if (currentService?.serverlessEnabled) { - const selectedServerIds = replicas - .filter((replica) => replica.count > 0) - .map((replica) => replica.serverId); + if (!currentService) throw new Error("Service not found"); + if (placement.mode === "automatic") { + if (currentService.serverlessEnabled) + throw new Error( + "Automatic placement is not supported for serverless services", + ); + const volume = await tx + .select({ id: serviceVolumes.id }) + .from(serviceVolumes) + .where(eq(serviceVolumes.serviceId, serviceId)) + .limit(1); + if (currentService.stateful || volume.length > 0) + throw new Error( + "Automatic placement is not supported for stateful services or services with volumes", + ); + await tx + .update(services) + .set({ placementMode: "automatic", replicas: placement.replicas }) + .where(eq(services.id, serviceId)); + await tx + .delete(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + return; + } + + const replicas = placement.placements; + const selectedServerIds = replicas.map((replica) => replica.serverId); + const selectedServers = await tx + .select({ + id: servers.id, + isProxy: servers.isProxy, + status: servers.status, + wireguardIp: servers.wireguardIp, + }) + .from(servers) + .where(inArray(servers.id, selectedServerIds)); + if (selectedServers.length !== selectedServerIds.length) + throw new Error("One or more selected servers do not exist"); + if ( + selectedServers.some( + (server) => server.status !== "online" || !server.wireguardIp, + ) + ) + throw new Error("Manual placement requires online, configured servers"); + if (currentService.serverlessEnabled) { if (selectedServerIds.length > 0) { - const workerServers = await tx - .select({ id: servers.id }) - .from(servers) - .where( - and( - inArray(servers.id, selectedServerIds), - eq(servers.isProxy, false), - ), - ); + const workerServers = selectedServers.filter( + (server) => !server.isProxy, + ); if (workerServers.length > 0) { throw new Error( "Disable serverless before deploying to worker nodes", @@ -1141,6 +1231,13 @@ export async function updateServiceConfig( } } + await tx + .update(services) + .set({ + placementMode: "manual", + replicas: replicas.reduce((sum, replica) => sum + replica.count, 0), + }) + .where(eq(services.id, serviceId)); await tx .delete(serviceReplicas) .where(eq(serviceReplicas.serviceId, serviceId)); @@ -1341,55 +1438,60 @@ export async function addServiceVolume( const validatedName = volumeNameSchema.parse(name); const validatedPath = containerPathSchema.parse(containerPath); - const service = await getService(serviceId); - if (!service) { - throw new Error("Service not found"); - } - - const configuredReplicas = await db - .select({ count: serviceReplicas.count }) - .from(serviceReplicas) - .where(eq(serviceReplicas.serviceId, serviceId)); - const totalReplicas = configuredReplicas.reduce( - (sum, r) => sum + r.count, - 0, - ); - - if (totalReplicas > 1) { - throw new Error( - "Volumes can only be added to services with 1 replica. Reduce replicas to 1 first.", + return await db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, ); - } - - const existing = await db - .select() - .from(serviceVolumes) - .where(eq(serviceVolumes.serviceId, serviceId)); + const service = await tx + .select() + .from(services) + .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) + .then((rows) => rows[0]); + if (!service) throw new Error("Service not found"); + if (service.placementMode === "automatic") { + throw new Error("Switch to manual placement before adding a volume"); + } - if (existing.some((v) => v.name === validatedName)) { - throw new Error("Volume with this name already exists"); - } + const configuredReplicas = await tx + .select({ count: serviceReplicas.count }) + .from(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + const totalReplicas = configuredReplicas.reduce( + (sum, replica) => sum + replica.count, + 0, + ); + if (totalReplicas > 1) { + throw new Error( + "Volumes can only be added to services with 1 replica. Reduce replicas to 1 first.", + ); + } - if (existing.some((v) => v.containerPath === validatedPath)) { - throw new Error("A volume with this container path already exists"); - } + const existing = await tx + .select() + .from(serviceVolumes) + .where(eq(serviceVolumes.serviceId, serviceId)); + if (existing.some((volume) => volume.name === validatedName)) { + throw new Error("Volume with this name already exists"); + } + if (existing.some((volume) => volume.containerPath === validatedPath)) { + throw new Error("A volume with this container path already exists"); + } - const id = randomUUID(); - await db.insert(serviceVolumes).values({ - id, - serviceId, - name: validatedName, - containerPath: validatedPath, + const id = randomUUID(); + await tx.insert(serviceVolumes).values({ + id, + serviceId, + name: validatedName, + containerPath: validatedPath, + }); + if (!service.stateful) { + await tx + .update(services) + .set({ stateful: true }) + .where(eq(services.id, serviceId)); + } + return { id, name: validatedName, containerPath: validatedPath }; }); - - if (!service.stateful) { - await db - .update(services) - .set({ stateful: true }) - .where(eq(services.id, serviceId)); - } - - return { id, name: validatedName, containerPath: validatedPath }; } catch (error) { if (error instanceof ZodError) { throw new Error( @@ -1421,7 +1523,9 @@ export async function removeServiceVolume(volumeId: string) { .from(deployments) .where(eq(deployments.serviceId, volume[0].serviceId)); - const hasRunning = activeDeployments.some(blocksProjectDeletion); + const hasRunning = activeDeployments.some(({ runtimeDesiredState }) => + isRuntimeExpected(runtimeDesiredState), + ); if (hasRunning) { throw new Error("Stop the service before removing volumes"); } diff --git a/web/actions/servers.ts b/web/actions/servers.ts index f0014f54..32452970 100644 --- a/web/actions/servers.ts +++ b/web/actions/servers.ts @@ -12,20 +12,12 @@ import { nameSchema } from "@/lib/schemas"; import type { DeleteConfirmation } from "@/lib/two-factor"; import { getZodErrorMessage } from "@/lib/utils"; -function generateId(): string { - return randomBytes(12).toString("hex"); -} - -function generateToken(): string { - return randomBytes(32).toString("hex"); -} - export async function createServer(name: string) { await requireDeveloperRole(); try { const validatedName = nameSchema.parse(name); - const id = generateId(); - const agentToken = generateToken(); + const id = randomBytes(12).toString("hex"); + const agentToken = randomBytes(32).toString("hex"); const now = new Date(); await db.insert(servers).values({ diff --git a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx index fe1131a1..77e9a995 100644 --- a/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx +++ b/web/app/(dashboard)/dashboard/projects/[slug]/[env]/services/[serviceId]/configuration/page.tsx @@ -20,10 +20,9 @@ import { SourceSection } from "@/components/service/details/source-section"; import { StartCommandSection } from "@/components/service/details/start-command-section"; import { VolumesSection } from "@/components/service/details/volumes-section"; import { useService } from "@/components/service/service-layout-client"; +import { observedReadyPhases } from "@/lib/deployment-status"; import type { DeleteConfirmation } from "@/lib/two-factor"; -const ACTIVE_DELETE_BACKUP_STATUSES = ["running", "healthy"] as const; - export default function ConfigurationPage() { const router = useRouter(); const { mutate: globalMutate } = useSWRConfig(); @@ -37,8 +36,8 @@ export default function ConfigurationPage() { } = useService(); const hasActiveDeploymentForBackup = service.deployments.some( (deployment) => - ACTIVE_DELETE_BACKUP_STATUSES.includes( - deployment.observedPhase as (typeof ACTIVE_DELETE_BACKUP_STATUSES)[number], + (observedReadyPhases as readonly string[]).includes( + deployment.observedPhase, ) && !!deployment.containerId, ); const hasVolumes = (service.volumes?.length ?? 0) > 0; diff --git a/web/app/api/cluster-health/route.ts b/web/app/api/cluster-health/route.ts index 98e92dc1..151ef276 100644 --- a/web/app/api/cluster-health/route.ts +++ b/web/app/api/cluster-health/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { auth } from "@/lib/auth"; import { headers } from "next/headers"; import { getClusterHealth } from "@/db/queries"; diff --git a/web/app/api/github/repos/route.ts b/web/app/api/github/repos/route.ts index 3b6a6ea6..6c3bff00 100644 --- a/web/app/api/github/repos/route.ts +++ b/web/app/api/github/repos/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { auth } from "@/lib/auth"; import { headers } from "next/headers"; import { db } from "@/db"; diff --git a/web/app/api/projects/[id]/environments/route.ts b/web/app/api/projects/[id]/environments/route.ts index 03206be1..0206ef21 100644 --- a/web/app/api/projects/[id]/environments/route.ts +++ b/web/app/api/projects/[id]/environments/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { auth } from "@/lib/auth"; import { headers } from "next/headers"; import { listEnvironments } from "@/db/queries"; diff --git a/web/app/api/projects/[id]/services/route.ts b/web/app/api/projects/[id]/services/route.ts index 5988f756..1137888d 100644 --- a/web/app/api/projects/[id]/services/route.ts +++ b/web/app/api/projects/[id]/services/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { and, desc, eq, inArray, isNull } from "drizzle-orm"; import { headers } from "next/headers"; import { db } from "@/db"; @@ -25,6 +23,7 @@ import { revisionSpecToDeployedConfig, type SourceConfig, } from "@/lib/service-config"; +import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; export async function GET( request: Request, @@ -154,22 +153,25 @@ export async function GET( .where(eq(serviceRevisions.id, activeDeployment.serviceRevisionId)) .then((rows) => rows[0]) : null; - const revisionServers = activeRevision + const activeSpecification = activeRevision + ? parseServiceRevisionSpec(activeRevision.specification) + : null; + const revisionServers = activeSpecification ? await db .select({ id: servers.id, name: servers.name }) .from(servers) .where( inArray( servers.id, - activeRevision.specification.placements.map( + activeSpecification.placements.map( (placement) => placement.serverId, ), ), ) : []; - const activeConfig = activeRevision + const activeConfig = activeSpecification ? revisionSpecToDeployedConfig( - activeRevision.specification, + activeSpecification, Object.fromEntries( revisionServers.map((server) => [server.id, server.name]), ), diff --git a/web/app/api/servers/route.ts b/web/app/api/servers/route.ts index 417077c1..126c4cd1 100644 --- a/web/app/api/servers/route.ts +++ b/web/app/api/servers/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { headers } from "next/headers"; import { db } from "@/db"; import { servers } from "@/db/schema"; diff --git a/web/app/api/services/[id]/revisions/route.ts b/web/app/api/services/[id]/revisions/route.ts index c11e7b08..c3f22b56 100644 --- a/web/app/api/services/[id]/revisions/route.ts +++ b/web/app/api/services/[id]/revisions/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { and, eq, isNull } from "drizzle-orm"; import { db } from "@/db"; import { services } from "@/db/schema"; diff --git a/web/app/api/services/[id]/rollouts/route.ts b/web/app/api/services/[id]/rollouts/route.ts index b5053a33..e997e9ed 100644 --- a/web/app/api/services/[id]/rollouts/route.ts +++ b/web/app/api/services/[id]/rollouts/route.ts @@ -1,12 +1,10 @@ -export const dynamic = "force-dynamic"; - -import { NextRequest, NextResponse } from "next/server"; -import { desc, eq } from "drizzle-orm"; +import { desc, eq, inArray } from "drizzle-orm"; +import { type NextRequest, NextResponse } from "next/server"; import { db } from "@/db"; -import { rollouts } from "@/db/schema"; +import { builds, rollouts } from "@/db/schema"; export async function GET( - request: NextRequest, + _request: NextRequest, { params }: { params: Promise<{ id: string }> }, ) { const { id: serviceId } = await params; @@ -18,5 +16,33 @@ export async function GET( .orderBy(desc(rollouts.createdAt)) .limit(50); - return NextResponse.json({ rollouts: rolloutsList }); + const buildRows = rolloutsList.length + ? await db + .select({ + serviceRevisionId: builds.serviceRevisionId, + commitSha: builds.commitSha, + commitMessage: builds.commitMessage, + }) + .from(builds) + .where( + inArray( + builds.serviceRevisionId, + rolloutsList.map((rollout) => rollout.serviceRevisionId), + ), + ) + : []; + const buildsByRevision = new Map( + buildRows.map((build) => [build.serviceRevisionId, build]), + ); + + return NextResponse.json({ + rollouts: rolloutsList.map((rollout) => { + const build = buildsByRevision.get(rollout.serviceRevisionId); + return { + ...rollout, + commitSha: build?.commitSha ?? null, + commitMessage: build?.commitMessage ?? null, + }; + }), + }); } diff --git a/web/app/api/services/[id]/secrets/[secretId]/reveal/route.ts b/web/app/api/services/[id]/secrets/[secretId]/reveal/route.ts index ff8ecbda..a42d2863 100644 --- a/web/app/api/services/[id]/secrets/[secretId]/reveal/route.ts +++ b/web/app/api/services/[id]/secrets/[secretId]/reveal/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { and, eq } from "drizzle-orm"; import { db } from "@/db"; import { secrets } from "@/db/schema"; diff --git a/web/app/api/services/[id]/secrets/route.ts b/web/app/api/services/[id]/secrets/route.ts index 248e136a..3ac004da 100644 --- a/web/app/api/services/[id]/secrets/route.ts +++ b/web/app/api/services/[id]/secrets/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { auth } from "@/lib/auth"; import { headers } from "next/headers"; import { db } from "@/db"; diff --git a/web/app/api/update-status/route.ts b/web/app/api/update-status/route.ts index e2c6a3cd..fe83803a 100644 --- a/web/app/api/update-status/route.ts +++ b/web/app/api/update-status/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { auth } from "@/lib/auth"; import { headers } from "next/headers"; import { refreshControlPlaneUpgradeState } from "@/lib/control-plane-updates"; diff --git a/web/app/api/v1/api-keys/route.ts b/web/app/api/v1/api-keys/route.ts index 1cdadf70..3d23b0a8 100644 --- a/web/app/api/v1/api-keys/route.ts +++ b/web/app/api/v1/api-keys/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { z } from "zod"; import { requireRequestRole } from "@/lib/api-auth"; import { auth as betterAuth } from "@/lib/auth"; diff --git a/web/app/api/v1/me/route.ts b/web/app/api/v1/me/route.ts index cb18138f..fe92f25f 100644 --- a/web/app/api/v1/me/route.ts +++ b/web/app/api/v1/me/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { requireApiKeyRole } from "@/lib/api-auth"; export async function GET(request: Request) { const auth = await requireApiKeyRole(request, [ diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/builds/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/builds/route.ts index 0f349105..d750b458 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/builds/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/builds/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { getBuilds as GET } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/configuration/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/configuration/route.ts index 5468d6c1..b4e71e9b 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/configuration/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/configuration/route.ts @@ -1,4 +1,3 @@ -export const dynamic = "force-dynamic"; export { getConfiguration as GET, patchConfigurationRoute as PATCH, diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/deploy/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/deploy/route.ts index 48fc6a50..e59b27b3 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/deploy/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/deploy/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { postDeploy as POST } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/logs/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/logs/route.ts index 84724297..1357e269 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/logs/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/logs/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { getServiceLogs as GET } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/metrics/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/metrics/route.ts index 18bcc9b5..12c1bca4 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/metrics/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/metrics/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { getMetrics as GET } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/revisions/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/revisions/route.ts index 840c2527..5278e9d1 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/revisions/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/revisions/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { getRevisions as GET } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/[rolloutId]/logs/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/[rolloutId]/logs/route.ts index e1ca946e..0ac21be3 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/[rolloutId]/logs/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/[rolloutId]/logs/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { getRolloutLogs as GET } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/[rolloutId]/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/[rolloutId]/route.ts index 7701dc70..6c594e0b 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/[rolloutId]/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/[rolloutId]/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { getRollout as GET } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/route.ts index 8a454b44..5412fd25 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/rollouts/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { getRollouts as GET } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/route.ts index 53272b0a..f696aa3c 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { requireApiKeyRole } from "@/lib/api-auth"; import { apiError, diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/status/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/status/route.ts index fdaa056c..56e8e302 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/status/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/[serviceId]/status/route.ts @@ -1,2 +1 @@ -export const dynamic = "force-dynamic"; export { getStatus as GET } from "@/lib/public-api-routes"; diff --git a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/route.ts b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/route.ts index 63cc3186..310cf593 100644 --- a/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/[environmentId]/services/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { and, asc, eq, gt, isNull, or } from "drizzle-orm"; import { db } from "@/db"; import { environments, services } from "@/db/schema"; diff --git a/web/app/api/v1/projects/[projectId]/environments/route.ts b/web/app/api/v1/projects/[projectId]/environments/route.ts index 1b99904b..ee66b75f 100644 --- a/web/app/api/v1/projects/[projectId]/environments/route.ts +++ b/web/app/api/v1/projects/[projectId]/environments/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { and, asc, eq, gt, or } from "drizzle-orm"; import { db } from "@/db"; import { environments, projects } from "@/db/schema"; diff --git a/web/app/api/v1/projects/route.ts b/web/app/api/v1/projects/route.ts index 4f5285c3..61a3fcda 100644 --- a/web/app/api/v1/projects/route.ts +++ b/web/app/api/v1/projects/route.ts @@ -1,5 +1,3 @@ -export const dynamic = "force-dynamic"; - import { and, asc, eq, gt, or } from "drizzle-orm"; import { db } from "@/db"; import { projects } from "@/db/schema"; diff --git a/web/components/auth/two-factor-challenge-page.tsx b/web/components/auth/two-factor-challenge-page.tsx index e23ee920..c5aa5e95 100644 --- a/web/components/auth/two-factor-challenge-page.tsx +++ b/web/components/auth/two-factor-challenge-page.tsx @@ -24,20 +24,11 @@ import { Label } from "@/components/ui/label"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; import { authClient, useSession } from "@/lib/auth-client"; -import { getSafeAuthRedirect } from "@/lib/two-factor"; - -type AuthClientError = { - message?: string; - error_description?: string; -} | null; - -function getErrorMessage(error: AuthClientError, fallback: string) { - return error?.message || error?.error_description || fallback; -} - -function normalizeCode(value: string) { - return value.replace(/\s/g, ""); -} +import { + getAuthErrorMessage, + getSafeAuthRedirect, + normalizeTwoFactorCode, +} from "@/lib/two-factor"; export function TwoFactorChallengePage() { const router = useRouter(); @@ -49,7 +40,7 @@ export function TwoFactorChallengePage() { const [trustDevice, setTrustDevice] = useState(false); const [error, setError] = useState(""); const [loading, setLoading] = useState(false); - const normalizedCode = useMemo(() => normalizeCode(code), [code]); + const normalizedCode = useMemo(() => normalizeTwoFactorCode(code), [code]); useEffect(() => { if (!isPending && session) { @@ -77,7 +68,7 @@ export function TwoFactorChallengePage() { if (response.error) { setError( - getErrorMessage( + getAuthErrorMessage( response.error, mode === "totp" ? "Invalid authenticator code" diff --git a/web/components/cluster/cluster-health-summary.tsx b/web/components/cluster/cluster-health-summary.tsx index fde95835..9f4b12e4 100644 --- a/web/components/cluster/cluster-health-summary.tsx +++ b/web/components/cluster/cluster-health-summary.tsx @@ -81,7 +81,7 @@ export function ClusterHealthSummary({

{stat.label}

-

+

{stat.value} {stat.subtitle} diff --git a/web/components/cluster/health-indicator.tsx b/web/components/cluster/health-indicator.tsx deleted file mode 100644 index 8694ba5a..00000000 --- a/web/components/cluster/health-indicator.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { cn } from "@/lib/utils"; -import type { ReactNode } from "react"; - -interface HealthIndicatorProps { - healthy: boolean | undefined | null; - label: string; - detail: string; - icon: ReactNode; -} - -export function HealthIndicator({ - healthy, - label, - detail, - icon, -}: HealthIndicatorProps) { - const isHealthy = healthy === true; - - return ( -

-
- {icon} -
-
-

{label}

-

{detail}

-
-
- ); -} diff --git a/web/components/cluster/resource-bar.tsx b/web/components/cluster/resource-bar.tsx deleted file mode 100644 index 2e8f590f..00000000 --- a/web/components/cluster/resource-bar.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { cn } from "@/lib/utils"; -import type { ReactNode } from "react"; - -interface ResourceBarProps { - value: number; - label: string; - icon: ReactNode; -} - -export function ResourceBar({ value, label, icon }: ResourceBarProps) { - const color = - value >= 90 - ? "bg-rose-500" - : value >= 70 - ? "bg-amber-500" - : "bg-emerald-500"; - - return ( -
-
- - {icon} - {label} - - {value.toFixed(1)}% -
-
-
-
-
- ); -} diff --git a/web/components/server/agent-update-nudge.tsx b/web/components/server/agent-update-nudge.tsx index 1223a98d..31189c20 100644 --- a/web/components/server/agent-update-nudge.tsx +++ b/web/components/server/agent-update-nudge.tsx @@ -93,9 +93,9 @@ export function AgentUpdateNudge({
- The control plane will send a signed work item to the agent. The - agent downloads the release binary, verifies its checksum, and - restarts itself after installation. + The control plane will send a work item over the authenticated + agent channel. The agent downloads the release binary, verifies + its checksum, and restarts itself after installation.
{isTargetUpgradeActive && ( diff --git a/web/components/service/details/deployment-progress.tsx b/web/components/service/details/deployment-progress.tsx index 87fb1742..239e5787 100644 --- a/web/components/service/details/deployment-progress.tsx +++ b/web/components/service/details/deployment-progress.tsx @@ -11,7 +11,10 @@ import type { DeploymentStatus, ServiceWithDetails as Service, } from "@/db/types"; -import type { ConfigChange } from "@/lib/service-config"; +import { + type ConfigChange, + getServiceTotalReplicas, +} from "@/lib/service-config"; import { cn } from "@/lib/utils"; type StageInfo = { @@ -150,10 +153,7 @@ export function getBarState( } } - const totalReplicas = service.configuredReplicas.reduce( - (sum, r) => sum + r.count, - 0, - ); + const totalReplicas = getServiceTotalReplicas(service); const hasNoDeployments = service.deployments.length === 0; const hasChanges = changes.length > 0; diff --git a/web/components/service/details/pending-changes-banner.tsx b/web/components/service/details/pending-changes-banner.tsx index 46cd2c03..4a48a802 100644 --- a/web/components/service/details/pending-changes-banner.tsx +++ b/web/components/service/details/pending-changes-banner.tsx @@ -11,6 +11,7 @@ import { Spinner } from "@/components/ui/spinner"; import type { ServiceWithDetails as Service } from "@/db/types"; import { type ConfigChange, + getServiceTotalReplicas, hasBuildAffectingChanges, } from "@/lib/service-config"; @@ -35,10 +36,7 @@ export const PendingChangesBanner = memo(function PendingChangesBanner({ const { mutate } = useSWRConfig(); const [isDeploying, setIsDeploying] = useState(false); - const totalReplicas = service.configuredReplicas.reduce( - (sum, r) => sum + r.count, - 0, - ); + const totalReplicas = getServiceTotalReplicas(service); const hasNoDeployments = service.deployments.length === 0; const shouldBuild = service.sourceType === "github" && diff --git a/web/components/service/details/replicas-section.tsx b/web/components/service/details/replicas-section.tsx index 935f2cde..280ddee3 100644 --- a/web/components/service/details/replicas-section.tsx +++ b/web/components/service/details/replicas-section.tsx @@ -14,21 +14,27 @@ import { } from "@/components/ui/empty"; import { Input } from "@/components/ui/input"; import { Spinner } from "@/components/ui/spinner"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import type { Server as ServerType, ServiceWithDetails as Service, } from "@/db/types"; -type ServerInfo = Pick; -type ServerWithStatus = ServerInfo & { status: string }; +type ServerInfo = Pick< + ServerType, + "id" | "name" | "isProxy" | "status" | "wireguardIp" +>; +type PlacementMode = "manual" | "automatic"; const fetcher = async (url: string): Promise => { const res = await fetch(url); - const servers: ServerWithStatus[] = await res.json(); - return servers.map(({ id, name, isProxy }) => ({ + const servers: ServerInfo[] = await res.json(); + return servers.map(({ id, name, isProxy, status, wireguardIp }) => ({ id, name, isProxy, + status, + wireguardIp, })); }; @@ -46,6 +52,11 @@ export const ReplicasSection = memo(function ReplicasSection({ {}, ); const [selectedServerId, setSelectedServerId] = useState(null); + const [placementMode, setPlacementMode] = useState( + service.serverlessEnabled ? "manual" : service.placementMode, + ); + const [desiredReplicas, setDesiredReplicas] = useState(service.replicas); + const [isEditing, setIsEditing] = useState(false); const [isSaving, setIsSaving] = useState(false); const configuredReplicas = useMemo( @@ -64,7 +75,7 @@ export const ReplicasSection = memo(function ReplicasSection({ } else { setSelectedServerId(null); } - } else { + } else if (!isEditing) { const replicaMap: Record = {}; for (const r of configuredReplicas) { replicaMap[r.serverId] = r.count; @@ -75,8 +86,68 @@ export const ReplicasSection = memo(function ReplicasSection({ } } setLocalReplicas(replicaMap); + setPlacementMode( + service.serverlessEnabled ? "manual" : service.placementMode, + ); + setDesiredReplicas(service.replicas); + } + }, [ + servers, + configuredReplicas, + service.stateful, + service.lockedServerId, + service.placementMode, + service.serverlessEnabled, + service.replicas, + isEditing, + ]); + + useEffect(() => { + if (!servers || service.stateful) return; + setLocalReplicas((current) => { + const next = { ...current }; + for (const server of servers) next[server.id] ??= 0; + return next; + }); + }, [servers, service.stateful]); + + useEffect(() => { + if ( + !isEditing || + service.stateful || + placementMode !== service.placementMode + ) + return; + if (placementMode === "automatic" && desiredReplicas === service.replicas) { + setIsEditing(false); + return; + } + if (placementMode === "manual") { + const configuredMap = new Map( + configuredReplicas.map((replica) => [replica.serverId, replica.count]), + ); + const localEntries = Object.entries(localReplicas).filter( + ([, count]) => count > 0, + ); + if ( + localEntries.length === configuredMap.size && + localEntries.every( + ([serverId, count]) => configuredMap.get(serverId) === count, + ) + ) { + setIsEditing(false); + } } - }, [servers, configuredReplicas, service.stateful, service.lockedServerId]); + }, [ + configuredReplicas, + desiredReplicas, + isEditing, + localReplicas, + placementMode, + service.placementMode, + service.replicas, + service.stateful, + ]); const hasChanges = useMemo(() => { if (service.stateful) { @@ -84,6 +155,10 @@ export const ReplicasSection = memo(function ReplicasSection({ configuredReplicas.length > 0 ? configuredReplicas[0].serverId : null; return selectedServerId !== currentServerId; } + if (placementMode !== service.placementMode) return true; + if (placementMode === "automatic") { + return desiredReplicas !== service.replicas; + } const configuredMap = new Map( configuredReplicas.map((r) => [r.serverId, r.count]), @@ -93,7 +168,16 @@ export const ReplicasSection = memo(function ReplicasSection({ if (configured !== count) return true; } return false; - }, [configuredReplicas, localReplicas, service.stateful, selectedServerId]); + }, [ + configuredReplicas, + desiredReplicas, + localReplicas, + placementMode, + service.placementMode, + service.replicas, + service.stateful, + selectedServerId, + ]); const isChangingServer = useMemo(() => { if (!service.stateful || !service.lockedServerId) return false; @@ -112,12 +196,28 @@ export const ReplicasSection = memo(function ReplicasSection({ ]); const updateReplicas = useCallback((serverId: string, value: number) => { + setIsEditing(true); setLocalReplicas((prev) => ({ ...prev, - [serverId]: Math.max(0, Math.min(10, value)), + [serverId]: Math.max(0, Math.min(10, Math.floor(value))), })); }, []); + const handleModeChange = (mode: string) => { + const nextMode = mode as PlacementMode; + if (nextMode === "automatic" && service.serverlessEnabled) return; + if (nextMode === placementMode) return; + setIsEditing(true); + if (nextMode === "automatic") { + const manualTotal = Object.values(localReplicas).reduce( + (sum, count) => sum + count, + 0, + ); + setDesiredReplicas(Math.max(1, Math.min(10, manualTotal || 1))); + } + setPlacementMode(nextMode); + }; + const handleSave = async () => { setIsSaving(true); try { @@ -126,12 +226,25 @@ export const ReplicasSection = memo(function ReplicasSection({ replicas = selectedServerId ? [{ serverId: selectedServerId, count: 1 }] : []; - } else { + } else if (placementMode === "manual") { replicas = Object.entries(localReplicas) .filter(([, count]) => count > 0) .map(([serverId, count]) => ({ serverId, count })); + await updateServiceConfig(service.id, { + placement: { mode: "manual", placements: replicas }, + }); + onUpdate(); + return; + } else { + await updateServiceConfig(service.id, { + placement: { mode: "automatic", replicas: desiredReplicas }, + }); + onUpdate(); + return; } - await updateServiceConfig(service.id, { replicas }); + await updateServiceConfig(service.id, { + placement: { mode: "manual", placements: replicas }, + }); onUpdate(); } finally { setIsSaving(false); @@ -148,11 +261,16 @@ export const ReplicasSection = memo(function ReplicasSection({ const workerUnavailableReason = service.serverlessEnabled ? "Disable serverless before deploying to worker nodes" : null; + const manualServerUnavailableReason = (server: ServerInfo) => { + if (server.status !== "online" || !server.wireguardIp) { + return "Server must be online and configured before placement"; + } + return service.serverlessEnabled && !server.isProxy + ? workerUnavailableReason + : null; + }; - const configuredTotal = configuredReplicas.reduce( - (sum, r) => sum + r.count, - 0, - ); + const manualTotalIsValid = totalReplicas >= 1 && totalReplicas <= 10; if (service.stateful) { return ( @@ -215,12 +333,8 @@ export const ReplicasSection = memo(function ReplicasSection({ type="button" key={server.id} onClick={() => setSelectedServerId(server.id)} - disabled={service.serverlessEnabled && !server.isProxy} - title={ - service.serverlessEnabled && !server.isProxy - ? workerUnavailableReason || undefined - : undefined - } + disabled={!!manualServerUnavailableReason(server)} + title={manualServerUnavailableReason(server) || undefined} className={`flex items-center gap-4 p-3 rounded-md text-left transition-colors ${ selectedServerId === server.id ? "bg-primary text-primary-foreground" @@ -264,15 +378,68 @@ export const ReplicasSection = memo(function ReplicasSection({ return ( 0 - ? `${configuredTotal} replica${configuredTotal !== 1 ? "s" : ""}` - : "No replicas" - } - summaryMuted={configuredTotal === 0} + summary={`${service.placementMode === "automatic" ? "Automatic" : "Manual"} · ${service.replicas} desired`} + summaryMuted={service.replicas === 0} >
- {isLoading ? ( + + + {!service.serverlessEnabled && ( + + Automatic + + )} + + Manual + + + + + {placementMode === "automatic" ? ( +
+
+ +

+ The control plane distributes replicas evenly across healthy + nodes and moves them after failures. +

+
+ { + setIsEditing(true); + setDesiredReplicas( + Math.max( + 1, + Math.min(10, Math.floor(event.target.valueAsNumber || 1)), + ), + ); + }} + className="w-24" + aria-describedby="automatic-replica-range" + /> +

+ Choose between 1 and 10 replicas. +

+ {hasChanges ? ( +
+ +
+ ) : null} +
+ ) : isLoading ? (
@@ -283,7 +450,8 @@ export const ReplicasSection = memo(function ReplicasSection({ No online servers available - Add a server to deploy this service. + Add a server to place replicas manually, or use automatic + placement. ) : ( @@ -292,11 +460,7 @@ export const ReplicasSection = memo(function ReplicasSection({ {servers.map((server) => (
@@ -316,14 +480,12 @@ export const ReplicasSection = memo(function ReplicasSection({ (localReplicas[server.id] || 0) - 1, ) } - disabled={ - (localReplicas[server.id] || 0) <= 0 || - (service.serverlessEnabled && !server.isProxy) - } + disabled={(localReplicas[server.id] || 0) <= 0} > - @@ -332,7 +494,7 @@ export const ReplicasSection = memo(function ReplicasSection({ parseInt(e.target.value, 10) || 0, ) } - disabled={service.serverlessEnabled && !server.isProxy} + disabled={!!manualServerUnavailableReason(server)} min={0} max={10} className="w-16 h-8 text-center" @@ -349,7 +511,7 @@ export const ReplicasSection = memo(function ReplicasSection({ } disabled={ (localReplicas[server.id] || 0) >= 10 || - (service.serverlessEnabled && !server.isProxy) + !!manualServerUnavailableReason(server) } > + @@ -364,14 +526,18 @@ export const ReplicasSection = memo(function ReplicasSection({ {totalReplicas !== 1 ? "s" : ""}
- {totalReplicas === 0 && ( + {!manualTotalIsValid && (

- Add at least 1 replica to deploy + Manual placement requires 1 to 10 replicas in total.

)} {hasChanges && (
-
diff --git a/web/components/service/details/rollout-history.tsx b/web/components/service/details/rollout-history.tsx index bfca6e66..bf8fa029 100644 --- a/web/components/service/details/rollout-history.tsx +++ b/web/components/service/details/rollout-history.tsx @@ -1,6 +1,13 @@ "use client"; -import { CheckCircle2, Clock, Loader2, RotateCcw, XCircle } from "lucide-react"; +import { + CheckCircle2, + Clock, + GitCommit, + Loader2, + RotateCcw, + XCircle, +} from "lucide-react"; import Link from "next/link"; import { useState } from "react"; import useSWR from "swr"; @@ -31,6 +38,8 @@ type RolloutListItem = { currentStage: string | null; createdAt: string; completedAt: string | null; + commitSha: string | null; + commitMessage: string | null; }; const STATUS_CONFIG: Record< @@ -201,15 +210,28 @@ export function RolloutHistory({ - - {rollout.status === "in_progress" - ? `Deploying — ${formatStage(rollout.currentStage)}` - : STATUS_TITLES[rollout.status]} - + {rollout.commitSha ? ( + <> + + + {rollout.commitSha.slice(0, 7)} + + + {rollout.commitMessage?.split("\n")[0] || + "No message"} + + + ) : ( + + {rollout.status === "in_progress" + ? `Deploying — ${formatStage(rollout.currentStage)}` + : STATUS_TITLES[rollout.status]} + + )} replica.count > 0 && !replica.serverIsProxy, ); - const unavailableReason = !hasPublicHttpEndpoint - ? "Add a public HTTP port with a domain to enable serverless" - : hasWorkerReplica - ? "Serverless services can only be deployed to proxy nodes" - : null; + const unavailableReason = + service.placementMode === "automatic" + ? "Switch to manual placement before enabling serverless" + : !hasPublicHttpEndpoint + ? "Add a public HTTP port with a domain to enable serverless" + : hasWorkerReplica + ? "Serverless services can only be deployed to proxy nodes" + : null; const optionsDisabled = !!unavailableReason || isSaving; const parsed = useMemo( diff --git a/web/components/service/service-canvas.tsx b/web/components/service/service-canvas.tsx index a2853a8b..0a02ad29 100644 --- a/web/components/service/service-canvas.tsx +++ b/web/components/service/service-canvas.tsx @@ -41,6 +41,7 @@ import { NativeSelectOption, } from "@/components/ui/native-select"; import type { Environment, ServiceWithDetails } from "@/db/types"; +import { observedReadyPhases } from "@/lib/deployment-status"; import { fetcher } from "@/lib/fetcher"; import { cn } from "@/lib/utils"; import { @@ -306,11 +307,11 @@ function ServiceCard({ p.isPublic && p.externalPort, ); - const hasInternalDns = service.deployments.some( - (d) => d.observedPhase === "running" || d.observedPhase === "healthy", + const hasInternalDns = service.deployments.some((d) => + (observedReadyPhases as readonly string[]).includes(d.observedPhase), ); - const runningCount = service.deployments.filter( - (d) => d.observedPhase === "running" || d.observedPhase === "healthy", + const runningCount = service.deployments.filter((d) => + (observedReadyPhases as readonly string[]).includes(d.observedPhase), ).length; const hasEndpoints = diff --git a/web/components/settings/edge-domain-settings.tsx b/web/components/settings/edge-domain-settings.tsx index 4368de33..110f6abf 100644 --- a/web/components/settings/edge-domain-settings.tsx +++ b/web/components/settings/edge-domain-settings.tsx @@ -114,10 +114,16 @@ export function EdgeDomainSettings({
- +

- Create one A record for each proxy IPv4 address using your DNS - provider and routing policy. + Configure these addresses as origins behind your external load + balancer. A stable external load balancer with active health + checks is the ideal production solution for proxy failure. +

+

+ A direct A record to one proxy has no ingress failover. Multiple A + records provide best-effort distribution, but clients may continue + using an offline proxy because of DNS caching.

{initial.hostname && ipv4Targets.length > 0 ? ( @@ -127,7 +133,7 @@ export function EdgeDomainSettings({ className="grid gap-1 px-3 py-2 text-sm sm:grid-cols-[4rem_1fr_1fr]" > - A + IP {initial.hostname} @@ -199,12 +205,14 @@ export function EdgeDomainSettings({

- Create wildcard DNS records for{" "} + Create a wildcard CNAME record for{" "} *. {initialAutoSubdomainDomain ?? "your automatic subdomain domain"} {" "} - that resolve to every proxy server public IPv4 address. + that points to the edge domain. In production, the edge domain + should resolve to a stable external load balancer with active health + checks.

diff --git a/web/components/settings/global-settings.tsx b/web/components/settings/global-settings.tsx index 63de5009..e131fd23 100644 --- a/web/components/settings/global-settings.tsx +++ b/web/components/settings/global-settings.tsx @@ -269,9 +269,6 @@ export function GlobalSettings({ Security - - API Keys - {membersData && ( Members @@ -451,9 +448,6 @@ export function GlobalSettings({ - - - diff --git a/web/components/settings/two-factor-settings.tsx b/web/components/settings/two-factor-settings.tsx index 550f0a95..27094e9f 100644 --- a/web/components/settings/two-factor-settings.tsx +++ b/web/components/settings/two-factor-settings.tsx @@ -25,12 +25,11 @@ import { Item, ItemContent, ItemMedia, ItemTitle } from "@/components/ui/item"; import { Label } from "@/components/ui/label"; import { Spinner } from "@/components/ui/spinner"; import { authClient, useSession } from "@/lib/auth-client"; -import { getTotpSecret } from "@/lib/two-factor"; - -type AuthClientError = { - message?: string; - error_description?: string; -} | null; +import { + getAuthErrorMessage, + getTotpSecret, + normalizeTwoFactorCode, +} from "@/lib/two-factor"; type TwoFactorSessionUser = { twoFactorEnabled?: boolean | null; @@ -42,14 +41,6 @@ type PendingSetup = { backupCodes: string[]; }; -function getErrorMessage(error: AuthClientError, fallback: string) { - return error?.message || error?.error_description || fallback; -} - -function normalizeCode(value: string) { - return value.replace(/\s/g, ""); -} - async function copyToClipboard(label: string, value: string) { try { await navigator.clipboard.writeText(value); @@ -120,7 +111,7 @@ export function TwoFactorSettings() { sessionUser?.twoFactorEnabled || recoveryCodes.length, ); const formattedVerificationCode = useMemo( - () => normalizeCode(verificationCode), + () => normalizeTwoFactorCode(verificationCode), [verificationCode], ); @@ -168,7 +159,7 @@ export function TwoFactorSettings() { if (response.error || !response.data?.totpURI) { throw new Error( - getErrorMessage(response.error, "Failed to start 2FA setup"), + getAuthErrorMessage(response.error, "Failed to start 2FA setup"), ); } @@ -202,7 +193,7 @@ export function TwoFactorSettings() { if (response.error) { throw new Error( - getErrorMessage( + getAuthErrorMessage( response.error, "Failed to verify authenticator code", ), @@ -238,7 +229,7 @@ export function TwoFactorSettings() { if (response.error || !response.data?.backupCodes) { throw new Error( - getErrorMessage(response.error, "Failed to generate backup codes"), + getAuthErrorMessage(response.error, "Failed to generate backup codes"), ); } @@ -267,7 +258,7 @@ export function TwoFactorSettings() { if (response.error || response.data?.status !== true) { throw new Error( - getErrorMessage(response.error, "Failed to disable 2FA"), + getAuthErrorMessage(response.error, "Failed to disable 2FA"), ); } diff --git a/web/components/ui/alert.tsx b/web/components/ui/alert.tsx index 1444df4a..2410a9f2 100644 --- a/web/components/ui/alert.tsx +++ b/web/components/ui/alert.tsx @@ -4,7 +4,7 @@ import { cva, type VariantProps } from "class-variance-authority"; import { cn } from "@/lib/utils"; const alertVariants = cva( - "grid gap-0.5 rounded-lg border px-2.5 py-2 text-left text-sm has-data-[slot=alert-action]:relative has-data-[slot=alert-action]:pr-18 has-[>svg]:grid-cols-[auto_1fr] has-[>svg]:gap-x-2 *:[svg]:row-span-2 *:[svg]:translate-y-0.5 *:[svg]:text-current *:[svg:not([class*='size-'])]:size-4 w-full relative group/alert", + "grid gap-0.5 rounded-lg border px-2.5 py-2 text-left text-sm has-[>svg]:grid-cols-[auto_1fr] has-[>svg]:gap-x-2 *:[svg]:row-span-2 *:[svg]:translate-y-0.5 *:[svg]:text-current *:[svg:not([class*='size-'])]:size-4 w-full relative group/alert", { variants: { variant: { @@ -63,14 +63,4 @@ function AlertDescription({ ); } -function AlertAction({ className, ...props }: React.ComponentProps<"div">) { - return ( -
- ); -} - -export { Alert, AlertTitle, AlertDescription, AlertAction }; +export { Alert, AlertTitle, AlertDescription }; diff --git a/web/components/ui/button-group.tsx b/web/components/ui/button-group.tsx index 64568546..411e0c16 100644 --- a/web/components/ui/button-group.tsx +++ b/web/components/ui/button-group.tsx @@ -1,77 +1,15 @@ -import { mergeProps } from "@base-ui/react/merge-props"; -import { useRender } from "@base-ui/react/use-render"; -import { cva, type VariantProps } from "class-variance-authority"; - import { cn } from "@/lib/utils"; -import { Separator } from "@/components/ui/separator"; - -const buttonGroupVariants = cva( - "has-[>[data-slot=button-group]]:gap-2 has-[select[aria-hidden=true]:last-child]:[&>[data-slot=select-trigger]:last-of-type]:rounded-r-lg flex w-fit items-stretch [&>*]:focus-visible:z-10 [&>*]:focus-visible:relative [&>[data-slot=select-trigger]:not([class*='w-'])]:w-fit [&>input]:flex-1", - { - variants: { - orientation: { - horizontal: - "[&>[data-slot]:not(:has(~[data-slot]))]:rounded-r-lg! [&>[data-slot]~[data-slot]]:rounded-l-none [&>[data-slot]~[data-slot]]:border-l-0 [&>[data-slot]]:rounded-r-none", - vertical: - "[&>[data-slot]:not(:has(~[data-slot]))]:rounded-b-lg! flex-col [&>[data-slot]~[data-slot]]:rounded-t-none [&>[data-slot]~[data-slot]]:border-t-0 [&>[data-slot]]:rounded-b-none", - }, - }, - defaultVariants: { - orientation: "horizontal", - }, - }, -); function ButtonGroup({ className, - orientation, ...props -}: React.ComponentProps<"div"> & VariantProps) { +}: React.ComponentProps<"div">) { return (
- ); -} - -function ButtonGroupText({ - className, - render, - ...props -}: useRender.ComponentProps<"div">) { - return useRender({ - defaultTagName: "div", - props: mergeProps<"div">( - { - className: cn( - "bg-muted gap-2 rounded-lg border px-2.5 text-sm font-medium [&_svg:not([class*='size-'])]:size-4 flex items-center [&_svg]:pointer-events-none", - className, - ), - }, - props, - ), - render, - state: { - slot: "button-group-text", - }, - }); -} - -function ButtonGroupSeparator({ - className, - orientation = "vertical", - ...props -}: React.ComponentProps) { - return ( - [data-slot=button-group]]:gap-2 has-[select[aria-hidden=true]:last-child]:[&>[data-slot=select-trigger]:last-of-type]:rounded-r-lg flex w-fit items-stretch [&>*]:focus-visible:z-10 [&>*]:focus-visible:relative [&>[data-slot=select-trigger]:not([class*='w-'])]:w-fit [&>input]:flex-1 [&>[data-slot]:not(:has(~[data-slot]))]:rounded-r-lg! [&>[data-slot]~[data-slot]]:rounded-l-none [&>[data-slot]~[data-slot]]:border-l-0 [&>[data-slot]]:rounded-r-none", className, )} {...props} @@ -79,9 +17,4 @@ function ButtonGroupSeparator({ ); } -export { - ButtonGroup, - ButtonGroupSeparator, - ButtonGroupText, - buttonGroupVariants, -}; +export { ButtonGroup }; diff --git a/web/components/ui/canvas-wrapper.tsx b/web/components/ui/canvas-wrapper.tsx index 7ed0baee..ce49bdfc 100644 --- a/web/components/ui/canvas-wrapper.tsx +++ b/web/components/ui/canvas-wrapper.tsx @@ -1,6 +1,7 @@ -"use client"; - -import { cn } from "@/lib/utils"; +import { + observedReadyPhases, + observedStartingPhases, +} from "@/lib/deployment-status"; export type StatusColors = { bg: string; @@ -74,22 +75,16 @@ export function getStatusColor(status: string): StatusColors { export function getStatusColorFromDeployments( deployments: { observedPhase: string; runtimeDesiredState?: string }[], ): StatusColors { - const hasRunning = deployments.some( - (d) => d.observedPhase === "running" || d.observedPhase === "healthy", + const hasRunning = deployments.some((d) => + (observedReadyPhases as readonly string[]).includes(d.observedPhase), ); - const hasPending = deployments.some( - (d) => - d.observedPhase === "pending" || - d.observedPhase === "pulling" || - d.observedPhase === "starting" || - d.observedPhase === "waking", + const hasPending = deployments.some((d) => + (observedStartingPhases as readonly string[]).includes(d.observedPhase), ); const hasFailed = deployments.some((d) => d.observedPhase === "failed"); const hasSleeping = deployments.some((d) => d.observedPhase === "sleeping"); const hasStopped = deployments.some( - (d) => - d.observedPhase === "stopped" || - d.runtimeDesiredState === "removed", + (d) => d.observedPhase === "stopped" || d.runtimeDesiredState === "removed", ); const hasUnknown = deployments.some((d) => d.observedPhase === "unknown"); @@ -101,57 +96,3 @@ export function getStatusColorFromDeployments( if (hasStopped) return statusColorMap.stopped; return defaultColors; } - -interface CanvasWrapperProps { - children?: React.ReactNode; - height?: string; - className?: string; - isEmpty?: boolean; - emptyContent?: React.ReactNode; -} - -export function CanvasWrapper({ - children, - height = "75vh", - className, - isEmpty, - emptyContent, -}: CanvasWrapperProps) { - if (isEmpty && emptyContent) { - return ( -
- {emptyContent} -
- ); - } - - return ( -
- {children} -
- ); -} diff --git a/web/components/ui/card.tsx b/web/components/ui/card.tsx index 43abce2a..2f18e051 100644 --- a/web/components/ui/card.tsx +++ b/web/components/ui/card.tsx @@ -25,7 +25,7 @@ function CardHeader({ className, ...props }: React.ComponentProps<"div">) {
) { ); } -function CardAction({ className, ...props }: React.ComponentProps<"div">) { - return ( -
- ); -} - function CardContent({ className, ...props }: React.ComponentProps<"div">) { return (
; } -function ContextMenuPortal({ ...props }: ContextMenuPrimitive.Portal.Props) { - return ( - - ); -} - function ContextMenuTrigger({ className, ...props @@ -63,32 +57,6 @@ function ContextMenuContent({ ); } -function ContextMenuGroup({ ...props }: ContextMenuPrimitive.Group.Props) { - return ( - - ); -} - -function ContextMenuLabel({ - className, - inset, - ...props -}: ContextMenuPrimitive.GroupLabel.Props & { - inset?: boolean; -}) { - return ( - - ); -} - function ContextMenuItem({ className, inset, @@ -155,67 +123,6 @@ function ContextMenuSubContent({ ); } -function ContextMenuCheckboxItem({ - className, - children, - checked, - ...props -}: ContextMenuPrimitive.CheckboxItem.Props) { - return ( - - - - - - - {children} - - ); -} - -function ContextMenuRadioGroup({ - ...props -}: ContextMenuPrimitive.RadioGroup.Props) { - return ( - - ); -} - -function ContextMenuRadioItem({ - className, - children, - ...props -}: ContextMenuPrimitive.RadioItem.Props) { - return ( - - - - - - - {children} - - ); -} - function ContextMenuSeparator({ className, ...props @@ -229,36 +136,13 @@ function ContextMenuSeparator({ ); } -function ContextMenuShortcut({ - className, - ...props -}: React.ComponentProps<"span">) { - return ( - - ); -} - export { ContextMenu, ContextMenuTrigger, ContextMenuContent, ContextMenuItem, - ContextMenuCheckboxItem, - ContextMenuRadioItem, - ContextMenuLabel, ContextMenuSeparator, - ContextMenuShortcut, - ContextMenuGroup, - ContextMenuPortal, ContextMenuSub, ContextMenuSubContent, ContextMenuSubTrigger, - ContextMenuRadioGroup, }; diff --git a/web/components/ui/dropdown-menu.tsx b/web/components/ui/dropdown-menu.tsx index f09f0c36..ce21f695 100644 --- a/web/components/ui/dropdown-menu.tsx +++ b/web/components/ui/dropdown-menu.tsx @@ -1,19 +1,14 @@ "use client"; -import * as React from "react"; import { Menu as MenuPrimitive } from "@base-ui/react/menu"; import { cn } from "@/lib/utils"; -import { ChevronRightIcon, CheckIcon } from "lucide-react"; +import { CheckIcon } from "lucide-react"; function DropdownMenu({ ...props }: MenuPrimitive.Root.Props) { return ; } -function DropdownMenuPortal({ ...props }: MenuPrimitive.Portal.Props) { - return ; -} - function DropdownMenuTrigger({ ...props }: MenuPrimitive.Trigger.Props) { return ; } @@ -99,58 +94,6 @@ function DropdownMenuItem({ ); } -function DropdownMenuSub({ ...props }: MenuPrimitive.SubmenuRoot.Props) { - return ; -} - -function DropdownMenuSubTrigger({ - className, - inset, - children, - ...props -}: MenuPrimitive.SubmenuTrigger.Props & { - inset?: boolean; -}) { - return ( - - {children} - - - ); -} - -function DropdownMenuSubContent({ - align = "start", - alignOffset = -3, - side = "right", - sideOffset = 0, - className, - ...props -}: React.ComponentProps) { - return ( - - ); -} - function DropdownMenuCheckboxItem({ className, children, @@ -229,25 +172,8 @@ function DropdownMenuSeparator({ ); } -function DropdownMenuShortcut({ - className, - ...props -}: React.ComponentProps<"span">) { - return ( - - ); -} - export { DropdownMenu, - DropdownMenuPortal, DropdownMenuTrigger, DropdownMenuContent, DropdownMenuGroup, @@ -257,8 +183,4 @@ export { DropdownMenuRadioGroup, DropdownMenuRadioItem, DropdownMenuSeparator, - DropdownMenuShortcut, - DropdownMenuSub, - DropdownMenuSubTrigger, - DropdownMenuSubContent, }; diff --git a/web/components/ui/empty.tsx b/web/components/ui/empty.tsx index 36165998..bcfd60bc 100644 --- a/web/components/ui/empty.tsx +++ b/web/components/ui/empty.tsx @@ -15,16 +15,6 @@ function Empty({ className, ...props }: React.ComponentProps<"div">) { ); } -function EmptyHeader({ className, ...props }: React.ComponentProps<"div">) { - return ( -
- ); -} - const emptyMediaVariants = cva( "mb-2 flex shrink-0 items-center justify-center [&_svg]:pointer-events-none [&_svg]:shrink-0", { @@ -93,7 +83,6 @@ function EmptyContent({ className, ...props }: React.ComponentProps<"div">) { export { Empty, - EmptyHeader, EmptyTitle, EmptyDescription, EmptyContent, diff --git a/web/components/ui/input-otp.tsx b/web/components/ui/input-otp.tsx index c2258750..afc0a016 100644 --- a/web/components/ui/input-otp.tsx +++ b/web/components/ui/input-otp.tsx @@ -1,7 +1,6 @@ "use client"; import { OTPInput, OTPInputContext } from "input-otp"; -import { MinusIcon } from "lucide-react"; import * as React from "react"; import { cn } from "@/lib/utils"; @@ -69,17 +68,4 @@ function InputOTPSlot({ ); } -function InputOTPSeparator({ ...props }: React.ComponentProps<"div">) { - return ( - - ); -} - -export { InputOTP, InputOTPGroup, InputOTPSlot, InputOTPSeparator }; +export { InputOTP, InputOTPGroup, InputOTPSlot }; diff --git a/web/components/ui/item.tsx b/web/components/ui/item.tsx index a5aa71cc..c1b2a162 100644 --- a/web/components/ui/item.tsx +++ b/web/components/ui/item.tsx @@ -2,7 +2,6 @@ import { mergeProps } from "@base-ui/react/merge-props"; import { useRender } from "@base-ui/react/use-render"; import { cva, type VariantProps } from "class-variance-authority"; import type * as React from "react"; -import { Separator } from "@/components/ui/separator"; import { cn } from "@/lib/utils"; function ItemGroup({ className, ...props }: React.ComponentProps<"div">) { @@ -19,20 +18,6 @@ function ItemGroup({ className, ...props }: React.ComponentProps<"div">) { ); } -function ItemSeparator({ - className, - ...props -}: React.ComponentProps) { - return ( - - ); -} - const itemVariants = cva( "[a]:hover:bg-muted rounded-lg border text-sm w-full group/item focus-visible:border-ring focus-visible:ring-ring/50 flex items-center flex-wrap outline-none transition-colors duration-100 focus-visible:ring-[3px] [a]:transition-colors", { @@ -165,41 +150,12 @@ function ItemActions({ className, ...props }: React.ComponentProps<"div">) { ); } -function ItemHeader({ className, ...props }: React.ComponentProps<"div">) { - return ( -
- ); -} - -function ItemFooter({ className, ...props }: React.ComponentProps<"div">) { - return ( -
- ); -} - export { Item, ItemMedia, ItemContent, ItemActions, ItemGroup, - ItemSeparator, ItemTitle, ItemDescription, - ItemHeader, - ItemFooter, }; diff --git a/web/components/ui/native-select.tsx b/web/components/ui/native-select.tsx index 9c79b12a..02a118f2 100644 --- a/web/components/ui/native-select.tsx +++ b/web/components/ui/native-select.tsx @@ -40,17 +40,4 @@ function NativeSelectOption({ ...props }: React.ComponentProps<"option">) { return - ); -} - -export { NativeSelect, NativeSelectOptGroup, NativeSelectOption }; +export { NativeSelect, NativeSelectOption }; diff --git a/web/components/ui/popover.tsx b/web/components/ui/popover.tsx index dfed9557..7cb0eab4 100644 --- a/web/components/ui/popover.tsx +++ b/web/components/ui/popover.tsx @@ -1,6 +1,5 @@ "use client"; -import * as React from "react"; import { Popover as PopoverPrimitive } from "@base-ui/react/popover"; import { cn } from "@/lib/utils"; @@ -47,44 +46,8 @@ function PopoverContent({ ); } -function PopoverHeader({ className, ...props }: React.ComponentProps<"div">) { - return ( -
- ); -} - -function PopoverTitle({ className, ...props }: PopoverPrimitive.Title.Props) { - return ( - - ); -} - -function PopoverDescription({ - className, - ...props -}: PopoverPrimitive.Description.Props) { - return ( - - ); -} - export { Popover, PopoverContent, - PopoverDescription, - PopoverHeader, - PopoverTitle, PopoverTrigger, }; diff --git a/web/components/ui/tabs.tsx b/web/components/ui/tabs.tsx index 5a2b5dd0..e4bcaa41 100644 --- a/web/components/ui/tabs.tsx +++ b/web/components/ui/tabs.tsx @@ -1,53 +1,31 @@ "use client"; import { Tabs as TabsPrimitive } from "@base-ui/react/tabs"; -import { cva, type VariantProps } from "class-variance-authority"; import { cn } from "@/lib/utils"; -function Tabs({ - className, - orientation = "horizontal", - ...props -}: TabsPrimitive.Root.Props) { +function Tabs({ className, ...props }: TabsPrimitive.Root.Props) { return ( ); } -const tabsListVariants = cva( - "rounded-lg p-[3px] group-data-horizontal/tabs:h-10 overflow-y-hidden data-[variant=line]:rounded-none group/tabs-list text-muted-foreground inline-flex w-fit items-center justify-center group-data-[orientation=vertical]/tabs:h-fit group-data-[orientation=vertical]/tabs:flex-col", - { - variants: { - variant: { - default: "bg-muted", - line: "gap-1 bg-transparent", - }, - }, - defaultVariants: { - variant: "default", - }, - }, -); - function TabsList({ className, - variant = "default", ...props -}: TabsPrimitive.List.Props & VariantProps) { +}: TabsPrimitive.List.Props) { return ( ); @@ -58,10 +36,8 @@ function TabsTrigger({ className, ...props }: TabsPrimitive.Tab.Props) { servers.id, { onDelete: "set null", diff --git a/web/db/types.ts b/web/db/types.ts index 809eedda..02bb6ca3 100644 --- a/web/db/types.ts +++ b/web/db/types.ts @@ -4,8 +4,6 @@ import type { deploymentPorts, deployments, environments, - githubInstallations, - githubRepos, memberInvitations, projects, rollouts, @@ -16,7 +14,6 @@ import type { services, serviceVolumes, user, - volumeBackups, workQueue, } from "./schema"; @@ -31,10 +28,7 @@ export type Secret = typeof secrets.$inferSelect; export type Deployment = typeof deployments.$inferSelect; export type DeploymentPort = typeof deploymentPorts.$inferSelect; export type Rollout = typeof rollouts.$inferSelect; -export type GithubRepo = typeof githubRepos.$inferSelect; -export type GithubInstallation = typeof githubInstallations.$inferSelect; export type Build = typeof builds.$inferSelect; -export type VolumeBackup = typeof volumeBackups.$inferSelect; export type WorkQueue = typeof workQueue.$inferSelect; export type User = typeof user.$inferSelect; export type MemberInvitation = typeof memberInvitations.$inferSelect; @@ -42,7 +36,6 @@ export type MemberRole = User["role"]; export type InvitableMemberRole = MemberInvitation["role"]; export type DeploymentStatus = NonNullable; -export type HealthStatus = Deployment["healthStatus"]; export type RolloutStatus = NonNullable; export type BuildStatus = NonNullable; diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index d4f3f291..6b19e2fb 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -23,6 +23,7 @@ import { isObservedReady, markDeploymentFailedRemoved, type ObservedPhase, + observedReadyPhases, observedStartingPhases, runtimeExpectedStates, } from "@/lib/deployment-status"; @@ -32,12 +33,12 @@ import { isRoutingSyncAcknowledgementEligible } from "@/lib/routing-sync"; import { getServerlessWakeFailureUpdate } from "@/lib/serverless-wake-failures"; import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; import { ingestRolloutLog } from "@/lib/victoria-logs"; -import { enqueueWork } from "@/lib/work-queue"; +import { enqueueWork, type WorkPayloadByType } from "@/lib/work-queue"; type ContainerStatus = { deploymentId: string; containerId: string; - status: "running" | "stopped" | "failed"; + status: "running" | "stopped" | "failed" | "transient"; healthStatus: "none" | "starting" | "healthy" | "unhealthy"; }; @@ -89,7 +90,7 @@ export function getStoppedContainerReportUpdate(deployment: { }; } -export function getStaleStoppedServerlessReportUpdate({ +export function getStaleStoppedReportUpdate({ hasHealthCheck, healthStatus, }: { @@ -331,7 +332,7 @@ async function applyServerlessTransitions( eq(deployments.serverId, serverId), eq(deployments.containerId, transition.containerId), eq(deployments.runtimeDesiredState, "running"), - inArray(deployments.observedPhase, ["healthy", "running"]), + inArray(deployments.observedPhase, observedReadyPhases), ), ) .returning({ id: deployments.id }); @@ -583,7 +584,7 @@ function getInvalidServerlessTransitionReason({ if (deployment.runtimeDesiredState !== "running") { return `deployment is not expected running (${deployment.runtimeDesiredState})`; } - if (!["healthy", "running"].includes(deployment.observedPhase)) { + if (!isObservedReady(deployment.observedPhase as ObservedPhase)) { return `deployment is not sleepable from ${deployment.observedPhase}`; } if (deployment.containerId !== transition.containerId) { @@ -779,6 +780,14 @@ export async function applyStatusReport( } for (const container of report.containers) { + // Transient containers (e.g. podman "created" mid-deploy) are reported + // for presence only — counted in reportedDeploymentIds above so the + // deployment isn't marked unknown or deleted, but their unsettled state + // must not drive any phase or health transition. + if (container.status === "transient") { + continue; + } + const healthStatus = container.healthStatus; let [deployment] = container.deploymentId @@ -872,9 +881,11 @@ export async function applyStatusReport( } const updateFields: Record = { healthStatus }; - let autohealRestartPayload: Record | null = null; - let autohealRecreatePayload: Record | null = null; + let autohealRestartPayload: WorkPayloadByType["restart"] | null = null; + let autohealRecreatePayload: WorkPayloadByType["force_cleanup"] | null = + null; let autohealFailed = false; + let restoredToReady = false; if (deployment.containerId !== container.containerId) { updateFields.containerId = container.containerId; @@ -921,16 +932,19 @@ export async function applyStatusReport( .where(eq(serviceRevisions.id, deployment.serviceRevisionId)) .then((r) => r[0]); - if (revision?.specification.serverless.enabled) { + if (revision) { Object.assign( updateFields, - getStaleStoppedServerlessReportUpdate({ + getStaleStoppedReportUpdate({ hasHealthCheck: revision.specification.healthCheck != null, healthStatus, }), ); + restoredToReady = + updateFields.observedPhase === "healthy" || + updateFields.observedPhase === "running"; console.log( - `[health:restore] serverless deployment ${deployment.id} restored from ${deployment.observedPhase} to ${updateFields.observedPhase}`, + `[health:restore] deployment ${deployment.id} restored from ${deployment.observedPhase} to ${updateFields.observedPhase}`, ); } } @@ -1009,6 +1023,7 @@ export async function applyStatusReport( ? "running" : "starting"; updateFields.observedPhase = newStatus; + restoredToReady = newStatus === "running"; console.log( `[health:restore] deployment ${deployment.id} restored from unknown to ${newStatus}`, ); @@ -1079,6 +1094,24 @@ export async function applyStatusReport( .set(updateFields) .where(eq(deployments.id, deployment.id)); + if (restoredToReady && deployment.rolloutId) { + const currentServerName = await getCurrentServerLogName(); + await ingestRolloutLog( + deployment.rolloutId, + deployment.serviceId, + "health_check", + `Container is healthy on server ${currentServerName}`, + ); + await inngest.send( + inngestEvents.resourceStatusChanged.create({ + type: "deployment", + id: deployment.id, + parentType: "rollout", + parentId: deployment.rolloutId, + }), + ); + } + if (autohealRestartPayload) { await enqueueWork(serverId, "restart", autohealRestartPayload); } @@ -1258,7 +1291,7 @@ function prepareAutohealRecreatePayload({ deployment: typeof deployments.$inferSelect; containerId: string; updateFields: Record; -}): Record | null { +}): WorkPayloadByType["force_cleanup"] | null { const decision = getSteadyStateRecreateDecision({ deployment, containerId }); Object.assign(updateFields, decision.updateFields); diff --git a/web/lib/agent-upgrades.ts b/web/lib/agent-upgrades.ts index 5fbef917..64801714 100644 --- a/web/lib/agent-upgrades.ts +++ b/web/lib/agent-upgrades.ts @@ -1,7 +1,8 @@ import { randomUUID } from "node:crypto"; -import { and, eq } from "drizzle-orm"; +import { eq } from "drizzle-orm"; import { db } from "@/db"; import { servers, workQueue } from "@/db/schema"; +import type { WorkPayloadByType } from "@/lib/work-queue"; const GITHUB_RELEASE_BASE_URL = "https://github.com/techulus/cloud/releases/download"; @@ -80,6 +81,10 @@ export async function enqueueAgentUpgrade( const arch = getReleaseArch(server.meta); const expectedSha256 = await fetchExpectedSha256(targetVersion, arch); + const payload = { + targetVersion, + expectedSha256, + } satisfies WorkPayloadByType["upgrade_agent"]; try { await db.transaction(async (tx) => { @@ -97,7 +102,7 @@ export async function enqueueAgentUpgrade( id: randomUUID(), serverId, type: "upgrade_agent", - payload: JSON.stringify({ targetVersion, expectedSha256 }), + payload: JSON.stringify(payload), }); }); } catch (error) { @@ -117,20 +122,3 @@ function isUniqueViolation(error: unknown) { (error as Error & { code?: string }).code === "23505" ); } - -export async function clearCompletedAgentUpgrade(serverId: string) { - await db - .update(servers) - .set({ - agentUpgradeTargetVersion: null, - agentUpgradeStatus: "idle", - agentUpgradeStartedAt: null, - agentUpgradeError: null, - }) - .where( - and( - eq(servers.id, serverId), - eq(servers.agentUpgradeStatus, "succeeded"), - ), - ); -} diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 0e4a0b5b..db0ada54 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -16,10 +16,10 @@ import { runtimeExpectedStates, } from "@/lib/deployment-status"; import { selectRoutingSyncRolloutIds } from "@/lib/routing-sync"; -import { - SERVICE_REVISION_SCHEMA_VERSION, - type ServiceRevisionSecret, - type ServiceRevisionSpec, +import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import type { + ServiceRevisionSecret, + ServiceRevisionSpec, } from "@/lib/service-revision-spec"; import { getWireGuardPeers } from "@/lib/wireguard"; @@ -241,9 +241,7 @@ async function getRuntimeServiceRevisions(): Promise { const runtimeServices = new Map(); for (const row of rows) { - if (row.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION) { - throw new Error(`Service ${row.serviceId} uses an unsupported revision`); - } + const specification = parseServiceRevisionSpec(row.specification); const existing = runtimeServices.get(row.serviceId); if (existing && existing.revisionId !== row.revisionId) { throw new Error(`Service ${row.serviceId} has multiple active revisions`); @@ -252,7 +250,7 @@ async function getRuntimeServiceRevisions(): Promise { id: row.serviceId, name: row.serviceName, revisionId: row.revisionId, - specification: row.specification, + specification, }); } return [...runtimeServices.values()].sort((a, b) => a.id.localeCompare(b.id)); @@ -271,10 +269,10 @@ async function buildExpectedContainers( ), ); - const serviceIds = unique(serverDeployments.map((dep) => dep.serviceId)); - const revisionIds = unique( - serverDeployments.map((dep) => dep.serviceRevisionId), - ); + const serviceIds = [...new Set(serverDeployments.map((dep) => dep.serviceId))]; + const revisionIds = [ + ...new Set(serverDeployments.map((dep) => dep.serviceRevisionId)), + ]; if (serviceIds.length === 0) return []; const [activeServices, revisions, depPorts] = await Promise.all([ @@ -329,7 +327,7 @@ export function buildExpectedContainersFromRows({ const servicesById = new Map( serviceRows.map((service) => [service.id, service]), ); - const portsByDeploymentId = groupBy( + const portsByDeploymentId = Map.groupBy( deploymentPortRows, (port) => port.deploymentId, ); @@ -352,14 +350,7 @@ export function buildExpectedContainersFromRows({ if (!revision) { throw new Error(`Deployment ${dep.id} has no service revision`); } - if ( - revision.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION - ) { - throw new Error( - `Deployment ${dep.id} uses an unsupported service revision`, - ); - } - const specification = revision.specification; + const specification = parseServiceRevisionSpec(revision.specification); const ports = (portsByDeploymentId.get(dep.id) ?? []) .slice() .sort( @@ -471,11 +462,11 @@ export function buildServerlessRoutesFromRows({ deployments: ServerlessDeploymentRow[]; containers: ExpectedContainer[]; }): ServerlessRoute[] { - const deploymentsByServiceId = groupBy( + const deploymentsByServiceId = Map.groupBy( deploymentRows, (deployment) => deployment.serviceId, ); - const portsByServiceId = groupBy(ports, (port) => port.serviceId); + const portsByServiceId = Map.groupBy(ports, (port) => port.serviceId); const expectedDeploymentIds = new Set( containers.map((container) => container.deploymentId), ); @@ -559,7 +550,7 @@ async function buildDnsRecords(allServices: RuntimeServiceRevision[]) { ), ); - const ipsByServiceId = groupBy( + const ipsByServiceId = Map.groupBy( dnsDeployments, (deployment) => deployment.serviceId, ); @@ -671,7 +662,7 @@ export function buildTraefikRoutes({ const httpRoutes: HttpRoute[] = []; const tcpRoutes: TcpRoute[] = []; const udpRoutes: UdpRoute[] = []; - const deploymentsByServiceId = groupBy( + const deploymentsByServiceId = Map.groupBy( routableDeployments, (deployment) => deployment.serviceId, ); @@ -843,20 +834,6 @@ function normalizeImage(image: string) { return image; } -function groupBy(items: T[], keyFn: (item: T) => K) { - const groups = new Map(); - for (const item of items) { - const key = keyFn(item); - const group = groups.get(key); - if (group) { - group.push(item); - } else { - groups.set(key, [item]); - } - } - return groups; -} - export function buildRuntimeRoutePorts( serviceRows: RuntimeServiceRevision[], ): RouteServicePort[] { @@ -881,7 +858,3 @@ function compareServicePorts(a: RouteServicePort, b: RouteServicePort) { a.port - b.port ); } - -function unique(items: T[]) { - return Array.from(new Set(items)); -} diff --git a/web/lib/backups/trigger-backup.ts b/web/lib/backups/trigger-backup.ts index abb79dda..5b44b83f 100644 --- a/web/lib/backups/trigger-backup.ts +++ b/web/lib/backups/trigger-backup.ts @@ -8,6 +8,7 @@ import { serviceVolumes, volumeBackups, } from "@/db/schema"; +import { observedReadyPhases } from "@/lib/deployment-status"; import { enqueueWork } from "@/lib/work-queue"; type TriggerBackupInput = { @@ -54,7 +55,7 @@ export async function triggerBackup({ .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.observedPhase, ["healthy", "running"]), + inArray(deployments.observedPhase, observedReadyPhases), ), ) .then((r) => r[0]); diff --git a/web/lib/build-assignment.ts b/web/lib/build-assignment.ts index 7239d1dc..1aea26aa 100644 --- a/web/lib/build-assignment.ts +++ b/web/lib/build-assignment.ts @@ -80,10 +80,12 @@ export async function selectBuildServerForRevision( const matchingServers = onlineServers.filter( (server) => server.meta?.arch === arch, ); - if (matchingServers.length === 0) { + const buildServers = + matchingServers.length > 0 ? matchingServers : onlineServers; + if (buildServers.length === 0) { throw new Error(`No online servers available for platform ${platform}`); } - return matchingServers[Math.floor(Math.random() * matchingServers.length)].id; + return buildServers[Math.floor(Math.random() * buildServers.length)].id; } export async function getTargetPlatformsForRevision( diff --git a/web/lib/cli-manifest.ts b/web/lib/cli-manifest.ts deleted file mode 100644 index 00382f62..00000000 --- a/web/lib/cli-manifest.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { z } from "zod"; -import { slugify } from "@/lib/utils"; - -const manifestPortSchema = z - .object({ - port: z.number().int().min(1).max(65535), - public: z.boolean().default(false), - domain: z.string().trim().min(1).optional(), - }) - .strict() - .superRefine((value, ctx) => { - if (value.public && !value.domain) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ["domain"], - message: "Public HTTP ports require a domain", - }); - } - - if (!value.public && value.domain) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: ["domain"], - message: "Internal ports cannot define a domain", - }); - } - }); - -const manifestHealthCheckSchema = z - .object({ - cmd: z.string().trim().min(1), - interval: z.number().int().min(1).default(10), - timeout: z.number().int().min(1).default(5), - retries: z.number().int().min(1).default(3), - startPeriod: z.number().int().min(0).default(30), - }) - .strict(); - -const manifestResourcesSchema = z - .object({ - cpuCores: z.number().min(0.1).max(64).nullable().optional(), - memoryMb: z.number().int().min(64).max(65536).nullable().optional(), - }) - .strict() - .superRefine((value, ctx) => { - const hasCpu = value.cpuCores !== undefined && value.cpuCores !== null; - const hasMemory = value.memoryMb !== undefined && value.memoryMb !== null; - - if (hasCpu !== hasMemory) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: "Resources must set both cpuCores and memoryMb together", - }); - } - }); - -export const techulusManifestSchema = z - .object({ - apiVersion: z.literal("v1"), - project: z.string().trim().min(1), - environment: z.string().trim().min(1), - service: z - .object({ - name: z.string().trim().min(1), - source: z - .object({ - type: z.literal("image"), - image: z.string().trim().min(1), - }) - .strict(), - hostname: z.string().trim().min(1).optional(), - ports: z.array(manifestPortSchema).default([]), - replicas: z - .object({ - count: z.number().int().min(1).max(10).default(1), - }) - .strict() - .default({ count: 1 }), - healthCheck: manifestHealthCheckSchema.optional(), - startCommand: z.string().trim().min(1).optional(), - resources: manifestResourcesSchema.optional(), - }) - .strict(), - }) - .strict(); - -export type TechulusManifest = z.infer; - -export function getManifestProjectSlug(manifest: TechulusManifest) { - return slugify(manifest.project); -} - -export function getManifestEnvironmentName(manifest: TechulusManifest) { - return slugify(manifest.environment); -} - -export function getManifestServiceName(manifest: TechulusManifest) { - return manifest.service.name.trim(); -} diff --git a/web/lib/deploy-service.ts b/web/lib/deploy-service.ts index e7825137..5384cad3 100644 --- a/web/lib/deploy-service.ts +++ b/web/lib/deploy-service.ts @@ -2,10 +2,9 @@ import { eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { db } from "@/db"; import { getService } from "@/db/queries"; -import { rollouts, serviceReplicas } from "@/db/schema"; -import { inngest } from "@/lib/inngest/client"; -import { inngestEvents } from "@/lib/inngest/events"; +import { serviceReplicas } from "@/db/schema"; import { startMigrationInternal } from "@/lib/migrations"; +import { sendRolloutCreated } from "@/lib/rollout-enqueue"; import type { ServiceRevisionActor } from "@/lib/service-revision-actor"; import { createRolloutForServiceRevision, @@ -25,17 +24,7 @@ export async function deployServiceRevisionInternal( ); if (!result.rolloutId) return result; - await inngest.send( - inngestEvents.rolloutCreated.create( - { - rolloutId: result.rolloutId, - serviceId, - }, - { - id: `rollout-created-${result.rolloutId}`, - }, - ), - ); + await sendRolloutCreated(result.rolloutId, serviceId); return result; } @@ -101,24 +90,7 @@ export async function deployServiceInternal( runtimeBaseRevisionId, ); - try { - await inngest.send( - inngestEvents.rolloutCreated.create({ - rolloutId, - serviceId, - }), - ); - } catch (error) { - await db - .update(rollouts) - .set({ - status: "failed", - currentStage: "enqueue_failed", - completedAt: new Date(), - }) - .where(eq(rollouts.id, rolloutId)); - throw error; - } + await sendRolloutCreated(rolloutId, serviceId); return { rolloutId }; } diff --git a/web/lib/inngest/functions/crons.ts b/web/lib/inngest/functions/crons.ts index 92c3a068..da5ef12c 100644 --- a/web/lib/inngest/functions/crons.ts +++ b/web/lib/inngest/functions/crons.ts @@ -10,6 +10,9 @@ import { checkAndRunScheduledDeployments, cleanupStaleItems, failTimedOutAgentUpgrades, + MAX_AUTOMATIC_RECOVERIES_PER_RUN, + rebalanceAutomaticServices, + recoverInvalidAutomaticPlacements, } from "@/lib/scheduler"; import { inngest } from "../client"; @@ -20,9 +23,25 @@ export const staleServerCheck = inngest.createFunction( singleton: { mode: "skip" }, }, async ({ step }) => { - await step.run("check-stale-servers", async () => { + const urgentCreated = await step.run("check-stale-servers", async () => { console.log("[cron] running stale server check"); - await checkAndRecoverStaleServers(); + return checkAndRecoverStaleServers(); + }); + const recoveredCreated = await step.run( + "recover-invalid-automatic-placements", + async () => { + return recoverInvalidAutomaticPlacements( + Math.max(0, MAX_AUTOMATIC_RECOVERIES_PER_RUN - urgentCreated), + ); + }, + ); + await step.run("rebalance-automatic-services", async () => { + await rebalanceAutomaticServices( + Math.max( + 0, + MAX_AUTOMATIC_RECOVERIES_PER_RUN - urgentCreated - recoveredCreated, + ), + ); }); }, ); diff --git a/web/lib/inngest/functions/restore-trigger-workflow.ts b/web/lib/inngest/functions/restore-trigger-workflow.ts index 6409cfa1..4ed9736b 100644 --- a/web/lib/inngest/functions/restore-trigger-workflow.ts +++ b/web/lib/inngest/functions/restore-trigger-workflow.ts @@ -2,6 +2,7 @@ import { and, eq, inArray } from "drizzle-orm"; import { db } from "@/db"; import { getBackupStorageConfig } from "@/db/queries"; import { deployments, volumeBackups } from "@/db/schema"; +import { observedReadyPhases } from "@/lib/deployment-status"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; import { inngestEvents } from "../events"; @@ -51,7 +52,7 @@ export const restoreTriggerWorkflow = inngest.createFunction( .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.observedPhase, ["healthy", "running"]), + inArray(deployments.observedPhase, observedReadyPhases), ), ) .then((r) => r[0]); diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index 3943b9f6..81b6c767 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, isNotNull } from "drizzle-orm"; import { db } from "@/db"; import { deploymentPorts, @@ -18,6 +18,38 @@ const PORT_RANGE_END = 32767; export type Placement = { serverId: string; replicas: number }; +export function automaticPlacementIneligibilityReason( + server: { + status: string; + wireguardIp: string | null; + isProxy: boolean; + }, + requireProxy = false, +): string | null { + if (server.status !== "online") return `status is ${server.status}`; + if (!server.wireguardIp) return "WireGuard is not configured"; + if (requireProxy && !server.isProxy) return "not a proxy node"; + return null; +} + +export function distributeReplicas( + serverIds: string[], + replicas: number, +): Placement[] { + const ids = [...new Set(serverIds)].sort((a, b) => a.localeCompare(b)); + if (ids.length === 0) throw new Error("No eligible servers for deployment"); + if (!Number.isInteger(replicas) || replicas < 1 || replicas > 10) + throw new Error("Replica count must be between 1 and 10"); + const counts = new Map(ids.map((id) => [id, 0])); + for (let index = 0; index < replicas; index++) { + const id = ids[index % ids.length]; + counts.set(id, (counts.get(id) ?? 0) + 1); + } + return ids + .map((serverId) => ({ serverId, replicas: counts.get(serverId) ?? 0 })) + .filter((placement) => placement.replicas > 0); +} + export type DeploymentContext = { revisionId: string; specification: ServiceRevisionSpec; @@ -30,16 +62,6 @@ export type DeploymentContext = { isRollingUpdate: boolean; }; -export function normalizeImage(image: string): string { - if (!image.includes("/")) { - return `docker.io/library/${image}`; - } - if (!image.includes(".") && image.split("/").length === 2) { - return `docker.io/${image}`; - } - return image; -} - async function getUsedPorts(serverId: string): Promise> { const existingPorts = await db .select({ hostPort: deploymentPorts.hostPort }) @@ -109,6 +131,74 @@ export function calculateRevisionPlacements( return { placements, totalReplicas }; } +export async function resolveRevisionPlacements( + specification: ServiceRevisionSpec, +): Promise<{ placements: Placement[]; totalReplicas: number }> { + if (specification.placement.mode === "manual") { + const result = calculateRevisionPlacements(specification); + if (specification.serverless.enabled) { + const selected = await db + .select({ id: servers.id, isProxy: servers.isProxy }) + .from(servers) + .where( + inArray( + servers.id, + result.placements.map((p) => p.serverId), + ), + ); + if ( + selected.length !== result.placements.length || + selected.some((server) => !server.isProxy) + ) + throw new Error( + "Serverless services can only be placed on proxy servers", + ); + } + return result; + } + const eligible = await db + .select({ id: servers.id }) + .from(servers) + .where( + and( + eq(servers.status, "online"), + isNotNull(servers.wireguardIp), + ...(specification.serverless.enabled + ? [eq(servers.isProxy, true)] + : []), + ), + ); + if (eligible.length === 0) { + const candidates = await db + .select({ + name: servers.name, + status: servers.status, + wireguardIp: servers.wireguardIp, + isProxy: servers.isProxy, + }) + .from(servers); + const details = candidates.length + ? candidates.map((server) => { + const reason = automaticPlacementIneligibilityReason( + server, + specification.serverless.enabled, + ); + return `${server.name}: ${reason ?? "eligible state changed during placement"}`; + }) + : ["no servers configured"]; + const message = `No eligible servers for deployment (${details.join("; ")})`; + console.warn(`[placement] ${message}`); + throw new Error(message); + } + return { + placements: distributeReplicas( + eligible.map((server) => server.id), + specification.placement.replicas, + ), + totalReplicas: specification.placement.replicas, + }; +} + export async function validateServers( placements: Placement[], ): Promise< @@ -295,7 +385,7 @@ export async function completeRollout( serviceId: string, context: Omit, ): Promise<{ completed: boolean; stoppedCount: number }> { - const { placements, specification, totalReplicas, isRollingUpdate } = context; + const { placements, specification, isRollingUpdate } = context; const lockedServerId = specification.stateful ? placements[0]?.serverId : undefined; @@ -327,13 +417,18 @@ export async function completeRollout( .returning({ id: deployments.id }) : []; - await tx - .update(services) - .set({ - replicas: totalReplicas, - ...(lockedServerId ? { lockedServerId } : {}), - }) - .where(eq(services.id, serviceId)); + const serviceUpdate = + specification.placement.mode === "automatic" + ? { lastAutomaticPlacementAt: new Date() } + : lockedServerId + ? { lockedServerId } + : null; + if (serviceUpdate) { + await tx + .update(services) + .set(serviceUpdate) + .where(eq(services.id, serviceId)); + } await tx .update(rollouts) diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index 96c8a1d7..cbf1aa4c 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -1,7 +1,8 @@ -import { and, eq, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; +import { and, eq, gte, inArray, isNull, lt, ne, or, sql } from "drizzle-orm"; import { db } from "@/db"; import { getService } from "@/db/queries"; import { deployments, rollouts, servers } from "@/db/schema"; +import { isObservedReady, observedReadyPhases } from "@/lib/deployment-status"; import { buildRoutingTargets } from "@/lib/routing-sync"; import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; import { getRolloutServiceRevision } from "@/lib/service-revisions"; @@ -9,13 +10,13 @@ import { ingestRolloutLog } from "@/lib/victoria-logs"; import { inngest } from "../client"; import { inngestEvents } from "../events"; import { - calculateRevisionPlacements, checkForRollingUpdate, cleanupExistingDeployments, cleanupTerminalDeployments, completeRollout, createDeploymentRecords, issueCertificatesForRevision, + resolveRevisionPlacements, validateServers, } from "./rollout-helpers"; import { handleRolloutFailure } from "./rollout-utils"; @@ -54,7 +55,7 @@ function getPreflightFailureReason(error: unknown) { return null; } -async function acquireRolloutTurn( +export async function acquireRolloutTurn( rolloutId: string, serviceId: string, ): Promise { @@ -62,7 +63,11 @@ async function acquireRolloutTurn( await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); const rollout = await tx - .select({ status: rollouts.status, createdAt: rollouts.createdAt }) + .select({ + status: rollouts.status, + currentStage: rollouts.currentStage, + createdAt: rollouts.createdAt, + }) .from(rollouts) .where(eq(rollouts.id, rolloutId)) .then((rows) => rows[0]); @@ -71,10 +76,41 @@ async function acquireRolloutTurn( throw new Error("Rollout not found"); } - if (rollout.status !== "queued") { + const recoverableEnqueueFailure = + rollout.status === "failed" && rollout.currentStage === "enqueue_failed"; + if (rollout.status !== "queued" && !recoverableEnqueueFailure) { return rollout.status === "in_progress" ? "acquired" : "terminal"; } + if (recoverableEnqueueFailure) { + const newerIntent = await tx + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + ne(rollouts.id, rolloutId), + gte(rollouts.createdAt, rollout.createdAt), + ), + ) + .limit(1) + .then((rows) => rows[0]); + + if (newerIntent) { + await tx + .update(rollouts) + .set({ currentStage: "superseded" }) + .where( + and( + eq(rollouts.id, rolloutId), + eq(rollouts.status, "failed"), + eq(rollouts.currentStage, "enqueue_failed"), + ), + ); + return "terminal"; + } + } + const blockingRollout = await tx .select({ id: rollouts.id }) .from(rollouts) @@ -104,6 +140,7 @@ async function acquireRolloutTurn( .set({ status: "in_progress", currentStage: "preparing", + completedAt: null, }) .where(eq(rollouts.id, rolloutId)); @@ -219,7 +256,7 @@ export const rolloutWorkflow = inngest.createFunction( const placementResult = await step.run("load-placements", async () => { try { - const result = calculateRevisionPlacements(specification); + const result = await resolveRevisionPlacements(specification); await ingestRolloutLog( rolloutId, serviceId, @@ -410,7 +447,7 @@ export const rolloutWorkflow = inngest.createFunction( .where( and( inArray(deployments.id, deploymentIds), - inArray(deployments.observedPhase, ["healthy", "running"]), + inArray(deployments.observedPhase, observedReadyPhases), ), ); @@ -447,9 +484,7 @@ export const rolloutWorkflow = inngest.createFunction( .where(inArray(deployments.id, pendingHealthDeploymentIds)); return deploymentStates.filter( - (deployment) => - deployment.observedPhase !== "healthy" && - deployment.observedPhase !== "running", + (deployment) => !isObservedReady(deployment.observedPhase), ); }, ); @@ -526,7 +561,7 @@ export const rolloutWorkflow = inngest.createFunction( and( eq(deployments.rolloutId, rolloutId), eq(deployments.trafficState, "candidate"), - inArray(deployments.observedPhase, ["healthy", "running"]), + inArray(deployments.observedPhase, observedReadyPhases), ), ); diff --git a/web/lib/inngest/functions/service-deletion-workflow.ts b/web/lib/inngest/functions/service-deletion-workflow.ts index 88c06bf6..13f4b05e 100644 --- a/web/lib/inngest/functions/service-deletion-workflow.ts +++ b/web/lib/inngest/functions/service-deletion-workflow.ts @@ -25,7 +25,11 @@ import { import { deleteBackupInternal } from "@/lib/backups/delete-backup"; import { addUtcDays, toDate } from "@/lib/date"; import { deployServiceInternal } from "@/lib/deploy-service"; -import { markDeploymentRemoved } from "@/lib/deployment-status"; +import { + isObservedReady, + markDeploymentRemoved, + observedReadyPhases, +} from "@/lib/deployment-status"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; import { enqueueWork } from "@/lib/work-queue"; import { inngest } from "../client"; @@ -84,7 +88,7 @@ export const serviceDeletionWorkflow = inngest.createFunction( .where( and( eq(deployments.serviceId, serviceId), - inArray(deployments.observedPhase, ["running", "healthy"]), + inArray(deployments.observedPhase, observedReadyPhases), ), ) .then((r) => r[0]); @@ -407,6 +411,9 @@ export const serviceRestoreWorkflow = inngest.createFunction( await step.run("restore-deletion-backups", async () => { for (const backup of setup.backups) { + if (!backup.storagePath || !backup.checksum) { + throw new Error("Backup data is incomplete"); + } await enqueueWork(setup.targetServerId, "restore_volume", { backupId: backup.id, serviceId, @@ -536,10 +543,8 @@ export const serviceRestoreWorkflow = inngest.createFunction( }, ); - const healthyDeployment = restoredDeployments.find( - (deployment) => - deployment.observedPhase === "healthy" || - deployment.observedPhase === "running", + const healthyDeployment = restoredDeployments.find((deployment) => + isObservedReady(deployment.observedPhase), ); const failedDeployment = restoredDeployments.find( (deployment) => deployment.observedPhase === "failed", diff --git a/web/lib/migrations.ts b/web/lib/migrations.ts index 20b51751..9ea6ef17 100644 --- a/web/lib/migrations.ts +++ b/web/lib/migrations.ts @@ -7,6 +7,7 @@ import { services, serviceVolumes, } from "@/db/schema"; +import { observedReadyPhases } from "@/lib/deployment-status"; import { inngest } from "@/lib/inngest/client"; import { inngestEvents } from "@/lib/inngest/events"; import type { ServiceRevisionActor } from "@/lib/service-revision-actor"; @@ -63,7 +64,7 @@ export async function startMigrationInternal( and( eq(deployments.serviceId, serviceId), eq(deployments.trafficState, "active"), - inArray(deployments.observedPhase, ["healthy", "running"]), + inArray(deployments.observedPhase, observedReadyPhases), ), ) .then((r) => r[0]); diff --git a/web/lib/project-deletion.ts b/web/lib/project-deletion.ts deleted file mode 100644 index 5b11f732..00000000 --- a/web/lib/project-deletion.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { deployments } from "@/db/schema"; - -type ProjectDeletionDeployment = Pick< - typeof deployments.$inferSelect, - "runtimeDesiredState" ->; - -export function blocksProjectDeletion(deployment: ProjectDeletionDeployment) { - // Sleeping serverless deployments still have runtime intent and can wake. - return deployment.runtimeDesiredState !== "removed"; -} diff --git a/web/lib/public-api.ts b/web/lib/public-api.ts index c782adf7..5f80af4f 100644 --- a/web/lib/public-api.ts +++ b/web/lib/public-api.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, desc, eq, isNull, ne } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, ne, sql } from "drizzle-orm"; import { z } from "zod"; import { db } from "@/db"; import { @@ -15,8 +15,12 @@ import { serviceVolumes, } from "@/db/schema"; import { validateDockerImageInternal } from "@/lib/docker-image"; +import { getServiceTotalReplicas } from "@/lib/service-config"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; -import { getDefaultServiceHostname } from "@/lib/service-revision-spec"; +import { + getDefaultServiceHostname, + getServiceRevisionTotalReplicas, +} from "@/lib/service-revision-spec"; const githubPathPart = /^[A-Za-z0-9_.-]+$/; const windowsAbsolutePath = /^[A-Za-z]:[\\/]/; @@ -219,7 +223,6 @@ function getManagementBlockers(input: { source: PublicSource; ports: Array; volumeCount: number; - replicaCount: number; }): ManagementBlocker[] { const blockers: ManagementBlocker[] = []; if (input.service.stateful || input.volumeCount > 0) { @@ -252,12 +255,6 @@ function getManagementBlockers(input: { "Public HTTP ports without domains must be managed in the web UI", }); } - if (input.replicaCount < 1 || input.replicaCount > 10) { - blockers.push({ - code: "INVALID_PLACEMENT", - message: "Manual placement must contain between 1 and 10 replicas", - }); - } if ( (input.service.resourceCpuLimit === null) !== (input.service.resourceMemoryLimitMb === null) @@ -286,6 +283,11 @@ function getManagementBlockers(input: { function sanitizeSpec(specification: unknown) { const spec = parseServiceRevisionSpec(specification); + const replicas = getServiceRevisionTotalReplicas(spec); + const placement = + spec.placement.mode === "automatic" + ? { mode: "automatic" as const, replicas } + : { mode: "manual" as const, placements: spec.placements, replicas }; return { source: spec.source.type === "github" @@ -293,17 +295,13 @@ function sanitizeSpec(specification: unknown) { type: "github" as const, repository: spec.source.repository, branch: spec.source.branch, - ...(spec.source.rootDir - ? { rootDir: spec.source.rootDir } - : {}), + ...(spec.source.rootDir ? { rootDir: spec.source.rootDir } : {}), } : { type: "image" as const, image: spec.source.image }, hostname: spec.hostname, stateful: spec.stateful, - replicas: spec.placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ), + placement, + replicas, placements: spec.placements, healthCheck: spec.healthCheck, startCommand: spec.startCommand, @@ -382,16 +380,25 @@ export async function safeConfiguration(service: NestedService) { a.name.localeCompare(b.name, "en") || a.containerPath.localeCompare(b.containerPath, "en"), ); - const replicaCount = sortedPlacements.reduce( - (sum, placement) => sum + placement.count, - 0, - ); + const replicaCount = getServiceTotalReplicas({ + ...service, + configuredReplicas: sortedPlacements, + }); + const placement = + service.placementMode === "automatic" + ? { mode: "automatic" as const, replicas: replicaCount } + : { + mode: "manual" as const, + placements: sortedPlacements, + replicas: replicaCount, + }; const current = { source, hostname: service.hostname, stateful: service.stateful, replicas: replicaCount, placements: sortedPlacements, + placement, healthCheck: service.healthCheckCmd ? { cmd: service.healthCheckCmd, @@ -454,11 +461,15 @@ export async function safeConfiguration(service: NestedService) { hostname: current.hostname?.trim() || getDefaultServiceHostname(service.name), stateful: current.stateful, - replicas: current.replicas, - placements: current.placements.map(({ serverId, count }) => ({ - serverId, - count, - })), + placement: + current.placement.mode === "automatic" + ? current.placement + : { + ...current.placement, + placements: current.placement.placements.map( + ({ serverId, count }) => ({ serverId, count }), + ), + }, healthCheck: current.healthCheck, startCommand: current.startCommand?.trim() || null, resources: current.resources, @@ -486,7 +497,6 @@ export async function safeConfiguration(service: NestedService) { source, ports, volumeCount: volumes.length, - replicaCount, }); return { @@ -546,11 +556,46 @@ const hostnameSchema = z /^[a-z0-9]+(?:-[a-z0-9]+)*$/, "hostname must contain only lowercase letters, numbers, and hyphens", ); +export const placementSchema = z.discriminatedUnion("mode", [ + z.strictObject({ + mode: z.literal("automatic"), + replicas: z.number().int().min(1).max(10), + }), + z + .strictObject({ + mode: z.literal("manual"), + placements: z + .array( + z.strictObject({ + serverId: z.string().min(1), + count: z.number().int().min(1).max(10), + }), + ) + .min(1), + }) + .superRefine((value, context) => { + if ( + new Set(value.placements.map((item) => item.serverId)).size !== + value.placements.length + ) + context.addIssue({ + code: "custom", + message: "Server IDs must be unique", + path: ["placements"], + }); + if (value.placements.reduce((sum, item) => sum + item.count, 0) > 10) + context.addIssue({ + code: "custom", + message: "Total replicas must be between 1 and 10", + path: ["placements"], + }); + }), +]); export const configurationPatchSchema = z.strictObject({ source: publicSourceSchema.optional(), hostname: hostnameSchema.nullable().optional(), ports: z.array(portSchema).max(100).optional(), - replicas: z.number().int().min(1).max(10).optional(), + placement: placementSchema.optional(), healthCheck: healthCheckSchema.nullable().optional(), startCommand: z.string().trim().min(1).max(4096).nullable().optional(), resources: z @@ -610,6 +655,9 @@ export async function patchConfiguration( } return db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${service.id}))`, + ); const persisted = await tx .select() .from(services) @@ -638,17 +686,22 @@ export async function patchConfiguration( .limit(1) .then((rows) => rows[0]), ]); - const replicaCount = placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ); const source = resolvePersistedSourceFromRows(persisted, repo); + if ( + input.placement?.mode === "automatic" && + (persisted.stateful || persisted.serverlessEnabled || volumes.length > 0) + ) { + domainError( + "Automatic placement is not supported for stateful, serverless, or volume-backed services", + "AUTOMATIC_PLACEMENT_UNSUPPORTED", + 400, + ); + } const blockers = getManagementBlockers({ service: persisted, source, ports, volumeCount: volumes.length, - replicaCount, }); if (blockers[0]) { domainError(blockers[0].message, blockers[0].code); @@ -677,11 +730,42 @@ export async function patchConfiguration( ); } } - if (input.replicas !== undefined && input.replicas !== replicaCount) { - domainError( - `replicas must match the current manual placement of ${replicaCount}`, - "REPLICA_PLACEMENT_MISMATCH", - ); + if (input.placement?.mode === "manual") { + const ids = input.placement.placements.map((item) => item.serverId); + const selected = await tx + .select({ + id: servers.id, + isProxy: servers.isProxy, + status: servers.status, + wireguardIp: servers.wireguardIp, + }) + .from(servers) + .where(inArray(servers.id, ids)); + if (selected.length !== ids.length) + domainError( + "One or more selected servers do not exist", + "INVALID_PLACEMENT", + 400, + ); + if ( + selected.some( + (server) => server.status !== "online" || !server.wireguardIp, + ) + ) + domainError( + "Manual placement requires online, configured servers", + "INVALID_PLACEMENT", + 400, + ); + if ( + persisted.serverlessEnabled && + selected.some((server) => !server.isProxy) + ) + domainError( + "Serverless services can only be deployed to proxy nodes", + "SERVERLESS_PROXY_REQUIRED", + 400, + ); } if (input.hostname) { @@ -808,6 +892,50 @@ export async function patchConfiguration( ) { set.image = input.source.image; } + if (input.placement) { + const desiredReplicas = + input.placement.mode === "automatic" + ? input.placement.replicas + : input.placement.placements.reduce( + (sum, item) => sum + item.count, + 0, + ); + if ( + changed( + "placement", + persisted.placementMode === "automatic" + ? { mode: "automatic", replicas: persisted.replicas } + : { + mode: "manual", + placements: placements + .map(({ serverId, count }) => ({ serverId, count })) + .toSorted((a, b) => a.serverId.localeCompare(b.serverId)), + }, + input.placement.mode === "manual" + ? { + ...input.placement, + placements: input.placement.placements.toSorted((a, b) => + a.serverId.localeCompare(b.serverId), + ), + } + : input.placement, + ) + ) { + set.placementMode = input.placement.mode; + set.replicas = desiredReplicas; + await tx + .delete(serviceReplicas) + .where(eq(serviceReplicas.serviceId, service.id)); + if (input.placement.mode === "manual") + await tx.insert(serviceReplicas).values( + input.placement.placements.map((item) => ({ + id: randomUUID(), + serviceId: service.id, + ...item, + })), + ); + } + } if (input.source?.type === "github") { const effectiveBranch = repo?.deployBranch || diff --git a/web/lib/rollout-enqueue.ts b/web/lib/rollout-enqueue.ts new file mode 100644 index 00000000..9e02daef --- /dev/null +++ b/web/lib/rollout-enqueue.ts @@ -0,0 +1,29 @@ +import { and, eq } from "drizzle-orm"; +import { db } from "@/db"; +import { rollouts } from "@/db/schema"; +import { inngest } from "@/lib/inngest/client"; +import { inngestEvents } from "@/lib/inngest/events"; + +export async function sendRolloutCreated( + rolloutId: string, + serviceId: string, +): Promise { + try { + await inngest.send( + inngestEvents.rolloutCreated.create( + { rolloutId, serviceId }, + { id: `rollout-created-${rolloutId}` }, + ), + ); + } catch (error) { + await db + .update(rollouts) + .set({ + status: "failed", + currentStage: "enqueue_failed", + completedAt: new Date(), + }) + .where(and(eq(rollouts.id, rolloutId), eq(rollouts.status, "queued"))); + throw error; + } +} diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index d7c31ea0..b66b76fb 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -5,6 +5,7 @@ import { deployments, rollouts, servers, + serviceRevisions, services, workQueue, } from "@/db/schema"; @@ -20,17 +21,291 @@ import { sendManualRecoveryRequiredAlert, sendServerOfflineAlert, } from "@/lib/email"; +import { + distributeReplicas, + resolveRevisionPlacements, +} from "@/lib/inngest/functions/rollout-helpers"; +import { sendRolloutCreated } from "@/lib/rollout-enqueue"; +import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import { cloneActiveRevisionAndQueueSystemRollout } from "@/lib/service-revisions"; import { WORK_QUEUE_LEASE_DURATION_MS, WORK_QUEUE_MAX_ATTEMPTS, } from "@/lib/work-queue"; const STALE_THRESHOLD_MS = 75 * SECOND_IN_MILLISECONDS; +export const AUTOMATIC_PLACEMENT_COOLDOWN_MS = 30 * MINUTE_IN_MILLISECONDS; +export const MAX_REBALANCES_PER_RUN = 5; +export const MAX_AUTOMATIC_RECOVERIES_PER_RUN = 5; + +export async function rebalanceAutomaticServices( + maxCreated = MAX_REBALANCES_PER_RUN, +): Promise { + if (maxCreated <= 0) return 0; + const now = new Date(); + const candidates = await db + .select({ + id: services.id, + name: services.name, + lastAutomaticPlacementAt: services.lastAutomaticPlacementAt, + }) + .from(services) + .innerJoin(deployments, eq(deployments.serviceId, services.id)) + .innerJoin( + serviceRevisions, + eq(serviceRevisions.id, deployments.serviceRevisionId), + ) + .where( + and( + isNull(services.deletedAt), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + eq( + sql`${serviceRevisions.specification} -> 'placement' ->> 'mode'`, + "automatic", + ), + ), + ) + .groupBy(services.id, services.name, services.lastAutomaticPlacementAt) + .orderBy(services.id); + let queuedCount = 0; + for (const service of candidates) { + if (queuedCount >= maxCreated) break; + if ( + service.lastAutomaticPlacementAt && + now.getTime() - service.lastAutomaticPlacementAt.getTime() < + AUTOMATIC_PLACEMENT_COOLDOWN_MS + ) + continue; + const [pending, activeDeployments] = await Promise.all([ + db + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, service.id), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .limit(1) + .then((r) => r[0]), + db + .select({ + revisionId: deployments.serviceRevisionId, + specification: serviceRevisions.specification, + }) + .from(deployments) + .innerJoin( + serviceRevisions, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) + .where( + and( + eq(deployments.serviceId, service.id), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + ), + ) + .then((rows) => rows), + ]); + const activeRevisionIds = new Set( + activeDeployments.map((deployment) => deployment.revisionId), + ); + if (pending || activeRevisionIds.size !== 1) continue; + const active = activeDeployments[0]; + if (!active) continue; + const spec = parseServiceRevisionSpec(active.specification); + if (spec.stateful || spec.placement.mode !== "automatic") continue; + const eligible = await db + .select({ id: servers.id }) + .from(servers) + .where( + and( + eq(servers.status, "online"), + isNotNull(servers.wireguardIp), + ...(spec.serverless.enabled ? [eq(servers.isProxy, true)] : []), + ), + ); + if (!eligible.length) continue; + const ideal = distributeReplicas( + eligible.map((s) => s.id), + spec.placement.replicas, + ); + const desired = await db + .select({ + serverId: deployments.serverId, + count: sql`count(*)::int`, + }) + .from(deployments) + .where( + and( + eq(deployments.serviceId, service.id), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + ), + ) + .groupBy(deployments.serverId); + const normalize = ( + rows: Array<{ serverId: string; count?: number; replicas?: number }>, + ) => rows.map((r) => [r.serverId, r.count ?? r.replicas]).sort(); + if (JSON.stringify(normalize(ideal)) === JSON.stringify(normalize(desired))) + continue; + try { + const result = await cloneActiveRevisionAndQueueSystemRollout( + service.id, + active.revisionId, + ); + if (!result.created) continue; + queuedCount++; + await sendRolloutCreated(result.rolloutId, service.id); + } catch (error) { + console.error(`[scheduler] failed to rebalance ${service.name}`, error); + } + } + return queuedCount; +} + +export async function recoverInvalidAutomaticPlacements( + maxCreated = MAX_AUTOMATIC_RECOVERIES_PER_RUN, +): Promise { + if (maxCreated <= 0) return 0; + const candidateServices = await db + .select({ serviceId: deployments.serviceId }) + .from(deployments) + .innerJoin(services, eq(services.id, deployments.serviceId)) + .innerJoin( + serviceRevisions, + eq(serviceRevisions.id, deployments.serviceRevisionId), + ) + .where( + and( + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + isNull(services.deletedAt), + eq( + sql`${serviceRevisions.specification} -> 'placement' ->> 'mode'`, + "automatic", + ), + ), + ) + .groupBy(deployments.serviceId); + if (candidateServices.length === 0) return 0; + + const activeDeployments = await db + .select({ + serviceId: deployments.serviceId, + serviceName: services.name, + revisionId: deployments.serviceRevisionId, + specification: serviceRevisions.specification, + serverStatus: servers.status, + serverWireguardIp: servers.wireguardIp, + lastRecoveryAttemptAt: services.lastAutomaticRecoveryAttemptAt, + }) + .from(deployments) + .innerJoin(services, eq(services.id, deployments.serviceId)) + .innerJoin(servers, eq(servers.id, deployments.serverId)) + .innerJoin( + serviceRevisions, + eq(serviceRevisions.id, deployments.serviceRevisionId), + ) + .where( + and( + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + isNull(services.deletedAt), + inArray( + deployments.serviceId, + candidateServices.map((service) => service.serviceId), + ), + ), + ) + .orderBy(deployments.serviceId, deployments.id); + + const byService = new Map(); + for (const deployment of activeDeployments) { + const current = byService.get(deployment.serviceId) ?? []; + current.push(deployment); + byService.set(deployment.serviceId, current); + } + + const orderedServices = [...byService.entries()].sort( + ([serviceIdA, deploymentsA], [serviceIdB, deploymentsB]) => { + const attemptedAtA = deploymentsA[0]?.lastRecoveryAttemptAt?.getTime(); + const attemptedAtB = deploymentsB[0]?.lastRecoveryAttemptAt?.getTime(); + if (attemptedAtA === undefined && attemptedAtB !== undefined) return -1; + if (attemptedAtA !== undefined && attemptedAtB === undefined) return 1; + if (attemptedAtA !== attemptedAtB) + return (attemptedAtA ?? 0) - (attemptedAtB ?? 0); + return serviceIdA.localeCompare(serviceIdB); + }, + ); + let createdCount = 0; + for (const [serviceId, serviceDeployments] of orderedServices) { + if (createdCount >= maxCreated) break; + try { + const revisionIds = new Set( + serviceDeployments.map((deployment) => deployment.revisionId), + ); + if (revisionIds.size !== 1) { + console.error( + `[scheduler] skipping automatic recovery for ${serviceId}: multiple active revisions`, + ); + continue; + } + const active = serviceDeployments[0]; + if (!active) continue; + const specification = parseServiceRevisionSpec(active.specification); + if ( + specification.stateful || + specification.placement.mode !== "automatic" + ) + continue; + const hasInvalidPlacement = serviceDeployments.some( + (deployment) => + deployment.serverStatus !== "online" || !deployment.serverWireguardIp, + ); + if (!hasInvalidPlacement) continue; + + const pending = await db + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .limit(1) + .then((rows) => rows[0]); + if (pending) continue; + + await db + .update(services) + .set({ lastAutomaticRecoveryAttemptAt: new Date() }) + .where(eq(services.id, serviceId)); + await resolveRevisionPlacements(specification); + const result = await cloneActiveRevisionAndQueueSystemRollout( + serviceId, + active.revisionId, + ); + if (!result.created) continue; + createdCount++; + await sendRolloutCreated(result.rolloutId, serviceId); + } catch (error) { + console.error( + `[scheduler] failed level-triggered recovery for ${serviceDeployments[0]?.serviceName ?? serviceId}`, + error, + ); + } + } + return createdCount; +} async function triggerRecoveryForOfflineServers( offlineServerIds: string[], -): Promise { - if (offlineServerIds.length === 0) return; + maxCreated: number, +): Promise { + if (offlineServerIds.length === 0 || maxCreated <= 0) return 0; const affectedDeployments = await db .select({ @@ -40,10 +315,18 @@ async function triggerRecoveryForOfflineServers( serverPublicIp: servers.publicIp, serverWireguardIp: servers.wireguardIp, serviceName: services.name, + serviceId: services.id, + serviceRevisionId: deployments.serviceRevisionId, + specification: serviceRevisions.specification, + trafficState: deployments.trafficState, }) .from(deployments) .innerJoin(servers, eq(servers.id, deployments.serverId)) .innerJoin(services, eq(services.id, deployments.serviceId)) + .innerJoin( + serviceRevisions, + eq(serviceRevisions.id, deployments.serviceRevisionId), + ) .where( and( inArray(deployments.serverId, offlineServerIds), @@ -51,13 +334,65 @@ async function triggerRecoveryForOfflineServers( inArray(deployments.trafficState, ["candidate", "active"]), isNull(services.deletedAt), ), - ); + ) + .orderBy(deployments.serviceId, deployments.id); if (affectedDeployments.length === 0) { console.log( `[scheduler] ${offlineServerIds.length} server(s) went offline; no active replicas need manual recovery`, ); - return; + return 0; + } + const automaticActiveByService = new Map< + string, + (typeof affectedDeployments)[number] + >(); + const manualDeploymentIds = new Set(); + for (const deployment of affectedDeployments) { + try { + const specification = parseServiceRevisionSpec(deployment.specification); + if (specification.placement.mode === "manual") { + manualDeploymentIds.add(deployment.deploymentId); + continue; + } + if ( + !specification.stateful && + deployment.trafficState === "active" && + !automaticActiveByService.has(deployment.serviceId) + ) { + automaticActiveByService.set(deployment.serviceId, deployment); + } + } catch (error) { + console.error( + `[scheduler] cannot classify deployment ${deployment.deploymentId} for recovery`, + error, + ); + } + } + + let createdCount = 0; + for (const deployment of automaticActiveByService.values()) { + if (createdCount >= maxCreated) break; + try { + const specification = parseServiceRevisionSpec(deployment.specification); + await db + .update(services) + .set({ lastAutomaticRecoveryAttemptAt: new Date() }) + .where(eq(services.id, deployment.serviceId)); + await resolveRevisionPlacements(specification); + const queued = await cloneActiveRevisionAndQueueSystemRollout( + deployment.serviceId, + deployment.serviceRevisionId, + ); + if (!queued.created) continue; + createdCount++; + await sendRolloutCreated(queued.rolloutId, deployment.serviceId); + } catch (error) { + console.error( + `[scheduler] automatic recovery failed for ${deployment.serviceName}; periodic recovery will retry`, + error, + ); + } } const affectedByServer = new Map< @@ -71,6 +406,7 @@ async function triggerRecoveryForOfflineServers( >(); for (const deployment of affectedDeployments) { + if (!manualDeploymentIds.has(deployment.deploymentId)) continue; const current = affectedByServer.get(deployment.serverId) ?? { serverName: deployment.serverName, serverIp: @@ -100,11 +436,12 @@ async function triggerRecoveryForOfflineServers( ); }); } + return createdCount; } export async function checkAndRecoverStaleServers( excludeServerId?: string, -): Promise { +): Promise { const staleThreshold = subtractMilliseconds(new Date(), STALE_THRESHOLD_MS); const conditions = [ @@ -127,7 +464,7 @@ export async function checkAndRecoverStaleServers( wireguardIp: servers.wireguardIp, }); - if (markedOffline.length === 0) return; + if (markedOffline.length === 0) return 0; const offlineIds = markedOffline.map((s) => s.id); console.log( @@ -146,9 +483,10 @@ export async function checkAndRecoverStaleServers( }); } - triggerRecoveryForOfflineServers(offlineIds).catch((error) => { - console.error("[scheduler] recovery failed:", error); - }); + return triggerRecoveryForOfflineServers( + offlineIds, + MAX_AUTOMATIC_RECOVERIES_PER_RUN, + ); } export async function checkAndRunScheduledDeployments(): Promise { diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index 62627391..51394867 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -1,4 +1,7 @@ -import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; +import { + getServiceRevisionTotalReplicas, + type ServiceRevisionSpec, +} from "@/lib/service-revision-spec"; export type ReplicaConfig = { serverId: string; @@ -50,6 +53,7 @@ export type ResourceLimitsConfig = { }; export type PlacementConfig = { + mode?: "manual" | "automatic"; replicas: number; }; @@ -97,6 +101,19 @@ export function hasBuildAffectingChanges(changes: ConfigChange[]): boolean { ); } +export function getServiceTotalReplicas(service: { + placementMode?: "manual" | "automatic" | null; + replicas: number; + configuredReplicas: Array<{ count: number }>; +}): number { + return service.placementMode === "automatic" + ? service.replicas + : service.configuredReplicas.reduce( + (sum, replica) => sum + replica.count, + 0, + ); +} + export function buildCurrentConfig( service: { image: string; @@ -110,6 +127,7 @@ export function buildCurrentConfig( resourceCpuLimit: number | null; resourceMemoryLimitMb: number | null; replicas: number; + placementMode?: "manual" | "automatic" | null; stateful?: boolean | null; serverlessEnabled?: boolean | null; serverlessSleepAfterSeconds?: number | null; @@ -129,13 +147,20 @@ export function buildCurrentConfig( ): DeployedConfig { const hasResourceLimits = service.resourceCpuLimit != null || service.resourceMemoryLimitMb != null; + const placementMode = service.placementMode ?? "manual"; + const replicaCount = getServiceTotalReplicas({ + ...service, + placementMode, + configuredReplicas: replicas, + }); return { source, hostname: service.hostname ?? undefined, stateful: service.stateful ?? false, placement: { - replicas: replicas.reduce((sum, r) => sum + r.count, 0), + mode: placementMode, + replicas: replicaCount, }, replicas: replicas.map((r) => ({ serverId: r.serverId, @@ -373,37 +398,62 @@ export function diffConfigs( }); } - const deployedReplicasMap = new Map( - (deployed.replicas || []).map((r) => [r.serverId, r]), - ); - const currentReplicasMap = new Map( - (current.replicas || []).map((r) => [r.serverId, r]), - ); - - for (const [serverId, currentReplica] of currentReplicasMap) { - const deployedReplica = deployedReplicasMap.get(serverId); - if (!deployedReplica) { - changes.push({ - field: `${currentReplica.serverName} replicas`, - from: "0", - to: String(currentReplica.count), - }); - } else if (deployedReplica.count !== currentReplica.count) { + const deployedPlacementMode = deployed.placement?.mode ?? "manual"; + const currentPlacementMode = current.placement?.mode ?? "manual"; + if (deployedPlacementMode !== currentPlacementMode) { + changes.push({ + field: "Placement mode", + from: deployedPlacementMode === "automatic" ? "Automatic" : "Manual", + to: currentPlacementMode === "automatic" ? "Automatic" : "Manual", + }); + } + if ( + deployedPlacementMode === "automatic" && + currentPlacementMode === "automatic" + ) { + if (deployed.placement?.replicas !== current.placement?.replicas) { changes.push({ - field: `${currentReplica.serverName} replicas`, - from: String(deployedReplica.count), - to: String(currentReplica.count), + field: "Desired replicas", + from: String(deployed.placement?.replicas ?? 0), + to: String(current.placement?.replicas ?? 0), }); } - } + } else if ( + deployedPlacementMode === "manual" && + currentPlacementMode === "manual" + ) { + const deployedReplicasMap = new Map( + (deployed.replicas || []).map((replica) => [replica.serverId, replica]), + ); + const currentReplicasMap = new Map( + (current.replicas || []).map((replica) => [replica.serverId, replica]), + ); + + for (const [serverId, currentReplica] of currentReplicasMap) { + const deployedReplica = deployedReplicasMap.get(serverId); + if (!deployedReplica) { + changes.push({ + field: `${currentReplica.serverName} replicas`, + from: "0", + to: String(currentReplica.count), + }); + } else if (deployedReplica.count !== currentReplica.count) { + changes.push({ + field: `${currentReplica.serverName} replicas`, + from: String(deployedReplica.count), + to: String(currentReplica.count), + }); + } + } - for (const [serverId, deployedReplica] of deployedReplicasMap) { - if (!currentReplicasMap.has(serverId)) { - changes.push({ - field: `${deployedReplica.serverName} replicas`, - from: String(deployedReplica.count), - to: "0 (removed)", - }); + for (const [serverId, deployedReplica] of deployedReplicasMap) { + if (!currentReplicasMap.has(serverId)) { + changes.push({ + field: `${deployedReplica.serverName} replicas`, + from: String(deployedReplica.count), + to: "0 (removed)", + }); + } } } @@ -645,6 +695,7 @@ export function revisionSpecToDeployedConfig( specification: ServiceRevisionSpec, serverNames: Record, ): DeployedConfig { + const replicaCount = getServiceRevisionTotalReplicas(specification); return { source: specification.source.type === "github" @@ -658,10 +709,8 @@ export function revisionSpecToDeployedConfig( hostname: specification.hostname, stateful: specification.stateful, placement: { - replicas: specification.placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ), + mode: specification.placement.mode, + replicas: replicaCount, }, replicas: specification.placements.map((placement) => ({ serverId: placement.serverId, diff --git a/web/lib/service-revision-changes.ts b/web/lib/service-revision-changes.ts index 5e3cd41f..21cd9000 100644 --- a/web/lib/service-revision-changes.ts +++ b/web/lib/service-revision-changes.ts @@ -7,8 +7,7 @@ import type { ServiceRevisionSpec, } from "@/lib/service-revision-spec"; -const serviceRevisionSpecSchema = z.strictObject({ - schemaVersion: z.literal(2), +const serviceRevisionSpecFields = { image: z.string(), source: z.discriminatedUnion("type", [ z.strictObject({ type: z.literal("image"), image: z.string() }), @@ -72,7 +71,46 @@ const serviceRevisionSpecSchema = z.strictObject({ volumes: z.array( z.strictObject({ name: z.string(), containerPath: z.string() }), ), +}; + +const serviceRevisionSpecV2Schema = z.strictObject({ + schemaVersion: z.literal(2), + ...serviceRevisionSpecFields, }); +const serviceRevisionSpecSchema = z + .strictObject({ + schemaVersion: z.literal(3), + placement: z.discriminatedUnion("mode", [ + z.strictObject({ mode: z.literal("manual") }), + z.strictObject({ + mode: z.literal("automatic"), + replicas: z.number().int().min(1).max(10), + }), + ]), + ...serviceRevisionSpecFields, + }) + .superRefine((spec, context) => { + if (spec.placement.mode === "automatic" && spec.placements.length > 0) + context.addIssue({ + code: "custom", + message: "Automatic placement snapshots cannot contain placements", + }); + if (spec.placement.mode === "automatic" && spec.volumes.length > 0) + context.addIssue({ + code: "custom", + message: "Services with volumes cannot use automatic placement", + }); + if (spec.stateful && spec.placement.mode === "automatic") + context.addIssue({ + code: "custom", + message: "Stateful services cannot use automatic placement", + }); + if (spec.serverless.enabled && spec.placement.mode === "automatic") + context.addIssue({ + code: "custom", + message: "Serverless services cannot use automatic placement", + }); + }); export type ServiceRevisionChange = { field: string; @@ -102,7 +140,12 @@ export type ServiceRevisionChangelogResponse = { }; export function parseServiceRevisionSpec(value: unknown): ServiceRevisionSpec { - return serviceRevisionSpecSchema.parse(value); + const version = (value as { schemaVersion?: unknown } | null)?.schemaVersion; + if (version === 2) { + const legacy = serviceRevisionSpecV2Schema.parse(value); + return { ...legacy, schemaVersion: 3, placement: { mode: "manual" } }; + } + return serviceRevisionSpecSchema.parse(value) as ServiceRevisionSpec; } function compareStrings(a: string, b: string): number { @@ -249,25 +292,44 @@ export function diffServiceRevisionSpecs( : `${current.resourceLimits.memoryMb} MB`, ); - const previousPlacements = new Map( - previous.placements.map((placement) => [placement.serverId, placement]), - ); - const currentPlacements = new Map( - current.placements.map((placement) => [placement.serverId, placement]), + add( + "Placement mode", + previous.placement.mode === "automatic" ? "Automatic" : "Manual", + current.placement.mode === "automatic" ? "Automatic" : "Manual", ); - for (const serverId of [ - ...new Set([...previousPlacements.keys(), ...currentPlacements.keys()]), - ].sort(compareStrings)) { - const before = previousPlacements.get(serverId); - const after = currentPlacements.get(serverId); - const serverName = serverNames.get(serverId)?.trim(); + if ( + previous.placement.mode === "automatic" && + current.placement.mode === "automatic" + ) { add( - serverName - ? `${serverName} replicas` - : `Deleted server (${serverId.slice(0, 8)}) replicas`, - before ? `${before.count} replicas` : "(none)", - after ? `${after.count} replicas` : "(removed)", + "Desired replicas", + String(previous.placement.replicas), + String(current.placement.replicas), + ); + } else if ( + previous.placement.mode === "manual" && + current.placement.mode === "manual" + ) { + const previousPlacements = new Map( + previous.placements.map((placement) => [placement.serverId, placement]), ); + const currentPlacements = new Map( + current.placements.map((placement) => [placement.serverId, placement]), + ); + for (const serverId of [ + ...new Set([...previousPlacements.keys(), ...currentPlacements.keys()]), + ].sort(compareStrings)) { + const before = previousPlacements.get(serverId); + const after = currentPlacements.get(serverId); + const serverName = serverNames.get(serverId)?.trim(); + add( + serverName + ? `${serverName} replicas` + : `Deleted server (${serverId.slice(0, 8)}) replicas`, + before ? `${before.count} replicas` : "(none)", + after ? `${after.count} replicas` : "(removed)", + ); + } } const previousPorts = new Map( diff --git a/web/lib/service-revision-spec.ts b/web/lib/service-revision-spec.ts index 3e44c3e7..5b289cb4 100644 --- a/web/lib/service-revision-spec.ts +++ b/web/lib/service-revision-spec.ts @@ -1,4 +1,4 @@ -export const SERVICE_REVISION_SCHEMA_VERSION = 2 as const; +export const SERVICE_REVISION_SCHEMA_VERSION = 3 as const; export function getDefaultServiceHostname(name: string): string { return name @@ -20,6 +20,10 @@ export type ServiceRevisionPlacement = { count: number; }; +export type ServiceRevisionPlacementIntent = + | { mode: "manual" } + | { mode: "automatic"; replicas: number }; + export type ServiceRevisionPort = { containerPort: number; isPublic: boolean; @@ -74,6 +78,7 @@ export type ServiceRevisionSpec = { cpuCores: number | null; memoryMb: number | null; }; + placement: ServiceRevisionPlacementIntent; placements: ServiceRevisionPlacement[]; ports: ServiceRevisionPort[]; secrets: ServiceRevisionSecret[]; @@ -97,6 +102,8 @@ export type ServiceRevisionDraft = { startCommand: string | null; resourceCpuLimit: number | null; resourceMemoryLimitMb: number | null; + placementMode?: "manual" | "automatic" | null; + replicas?: number; }; placements: Array<{ serverId: string; count: number }>; ports: Array<{ @@ -121,14 +128,22 @@ export type ServiceRevisionSpecOverrides = { allowNoPlacements?: boolean; }; +export function getServiceRevisionTotalReplicas( + specification: Pick, +): number { + return specification.placement.mode === "automatic" + ? specification.placement.replicas + : specification.placements.reduce( + (sum, placement) => sum + placement.count, + 0, + ); +} + function validateServiceRevisionSpec( specification: ServiceRevisionSpec, allowNoPlacements: boolean, ) { - const totalReplicas = specification.placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ); + const totalReplicas = getServiceRevisionTotalReplicas(specification); if (totalReplicas < 1 && !allowNoPlacements) { throw new Error("At least one replica is required"); @@ -136,6 +151,27 @@ function validateServiceRevisionSpec( if (totalReplicas > 10) { throw new Error("Maximum 10 replicas allowed"); } + if ( + specification.placement.mode === "automatic" && + specification.placements.length + ) { + throw new Error("Automatic placement snapshots cannot contain placements"); + } + if ( + specification.placement.mode === "automatic" && + specification.volumes.length > 0 + ) { + throw new Error("Services with volumes cannot use automatic placement"); + } + if (specification.stateful && specification.placement.mode === "automatic") { + throw new Error("Stateful services cannot use automatic placement"); + } + if ( + specification.serverless.enabled && + specification.placement.mode === "automatic" + ) { + throw new Error("Serverless services cannot use automatic placement"); + } if (specification.stateful && totalReplicas !== 1) { throw new Error("Stateful services can only have exactly 1 replica"); } @@ -195,7 +231,11 @@ export function buildServiceRevisionSpec( cpuCores: service.resourceCpuLimit, memoryMb: service.resourceMemoryLimitMb, }, - placements: draft.placements + placement: + service.placementMode === "automatic" + ? { mode: "automatic", replicas: service.replicas ?? 1 } + : { mode: "manual" }, + placements: (service.placementMode === "automatic" ? [] : draft.placements) .filter((placement) => placement.count > 0) .map((placement) => ({ serverId: placement.serverId, diff --git a/web/lib/service-revisions.ts b/web/lib/service-revisions.ts index cc05aa31..4ba35cfd 100644 --- a/web/lib/service-revisions.ts +++ b/web/lib/service-revisions.ts @@ -1,7 +1,8 @@ import { randomUUID } from "node:crypto"; -import { and, eq, isNull } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; import { db } from "@/db"; import { + deployments, githubRepos, rollouts, secrets, @@ -16,7 +17,6 @@ import type { ServiceRevisionActor } from "@/lib/service-revision-actor"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; import { buildServiceRevisionSpec, - SERVICE_REVISION_SCHEMA_VERSION, type ServiceRevisionSource, type ServiceRevisionSpec, type ServiceRevisionSpecOverrides, @@ -186,76 +186,74 @@ export async function createGitHubBuildServiceRevision(input: { expectedBranch: string; actor: ServiceRevisionActor | null; }) { - return db.transaction( - async (tx) => { - const existing = await tx - .select() - .from(serviceRevisions) - .where(eq(serviceRevisions.id, input.id)) - .then((rows) => rows[0]); - if (existing) { - assertMatchingGitHubBuildRevision(existing, input); - return existing; - } + return db.transaction(async (tx) => { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtext(${input.serviceId}))`, + ); + const existing = await tx + .select() + .from(serviceRevisions) + .where(eq(serviceRevisions.id, input.id)) + .then((rows) => rows[0]); + if (existing) { + assertMatchingGitHubBuildRevision(existing, input); + return existing; + } - const [service, repo] = await Promise.all([ - tx - .select() - .from(services) - .where( - and(eq(services.id, input.serviceId), isNull(services.deletedAt)), - ) - .then((rows) => rows[0]), - tx - .select() - .from(githubRepos) - .where(eq(githubRepos.serviceId, input.serviceId)) - .then((rows) => rows[0]), - ]); - if (!service || service.sourceType !== "github") { - throw new Error("Active GitHub service not found"); - } + const [service, repo] = await Promise.all([ + tx + .select() + .from(services) + .where( + and(eq(services.id, input.serviceId), isNull(services.deletedAt)), + ) + .then((rows) => rows[0]), + tx + .select() + .from(githubRepos) + .where(eq(githubRepos.serviceId, input.serviceId)) + .then((rows) => rows[0]), + ]); + if (!service || service.sourceType !== "github") { + throw new Error("Active GitHub service not found"); + } - const currentSource = resolvePersistedSourceFromRows(service, repo); - if ( - currentSource.type !== "github" || - !currentSource.repository || - currentSource.repository !== input.expectedRepository || - currentSource.branch !== input.expectedBranch - ) { - throw new Error( - "GitHub source changed while resolving the build commit", - ); - } + const currentSource = resolvePersistedSourceFromRows(service, repo); + if ( + currentSource.type !== "github" || + !currentSource.repository || + currentSource.repository !== input.expectedRepository || + currentSource.branch !== input.expectedBranch + ) { + throw new Error("GitHub source changed while resolving the build commit"); + } - const source: ServiceRevisionSource = { - type: "github", - repository: currentSource.repository, - repositoryId: repo?.repoId ?? null, - branch: currentSource.branch, - commitSha: input.commitSha, - rootDir: currentSource.rootDir?.trim() || null, - authentication: repo - ? { - type: "github_app", - installationId: repo.installationId, - } - : { type: "anonymous" }, - }; + const source: ServiceRevisionSource = { + type: "github", + repository: currentSource.repository, + repositoryId: repo?.repoId ?? null, + branch: currentSource.branch, + commitSha: input.commitSha, + rootDir: currentSource.rootDir?.trim() || null, + authentication: repo + ? { + type: "github_app", + installationId: repo.installationId, + } + : { type: "anonymous" }, + }; - return createServiceRevisionSnapshot(tx, { - id: input.id, - serviceId: input.serviceId, - actor: input.actor, - overrides: { - image: input.image, - source, - allowNoPlacements: true, - }, - }); - }, - { isolationLevel: "repeatable read" }, - ); + return createServiceRevisionSnapshot(tx, { + id: input.id, + serviceId: input.serviceId, + actor: input.actor, + overrides: { + image: input.image, + source, + allowNoPlacements: true, + }, + }); + }); } export async function cloneGitHubBuildServiceRevision(input: { @@ -318,52 +316,109 @@ export async function createRolloutWithServiceRevision( actor: ServiceRevisionActor | null, runtimeBaseRevisionId?: string, ) { - return db.transaction( - async (tx) => { - let overrides: ServiceRevisionSpecOverrides | undefined; - if (runtimeBaseRevisionId) { - const baseRevision = await tx - .select({ specification: serviceRevisions.specification }) - .from(serviceRevisions) - .where( - and( - eq(serviceRevisions.id, runtimeBaseRevisionId), - eq(serviceRevisions.serviceId, serviceId), - ), - ) - .then((rows) => rows[0]); - if (!baseRevision) { - throw new Error("Runtime base service revision not found"); - } - const baseSpecification = parseServiceRevisionSpec( - baseRevision.specification, - ); - if (baseSpecification.source.type !== "github") { - throw new Error("GitHub runtime base revision is not a GitHub build"); - } - overrides = { - image: baseSpecification.image, - source: baseSpecification.source, - }; + return db.transaction(async (tx) => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${serviceId}))`); + let overrides: ServiceRevisionSpecOverrides | undefined; + if (runtimeBaseRevisionId) { + const baseRevision = await tx + .select({ specification: serviceRevisions.specification }) + .from(serviceRevisions) + .where( + and( + eq(serviceRevisions.id, runtimeBaseRevisionId), + eq(serviceRevisions.serviceId, serviceId), + ), + ) + .then((rows) => rows[0]); + if (!baseRevision) { + throw new Error("Runtime base service revision not found"); } - const revision = await createServiceRevisionSnapshot(tx, { - id: randomUUID(), - serviceId, - actor, - overrides, - }); - const rolloutId = randomUUID(); - await tx.insert(rollouts).values({ - id: rolloutId, - serviceId, - serviceRevisionId: revision.id, - status: "queued", - currentStage: "queued", - }); - return { rolloutId, revision }; - }, - { isolationLevel: "repeatable read" }, - ); + const baseSpecification = parseServiceRevisionSpec( + baseRevision.specification, + ); + if (baseSpecification.source.type !== "github") { + throw new Error("GitHub runtime base revision is not a GitHub build"); + } + overrides = { + image: baseSpecification.image, + source: baseSpecification.source, + }; + } + const revision = await createServiceRevisionSnapshot(tx, { + id: randomUUID(), + serviceId, + actor, + overrides, + }); + const rolloutId = randomUUID(); + await tx.insert(rollouts).values({ + id: rolloutId, + serviceId, + serviceRevisionId: revision.id, + status: "queued", + currentStage: "queued", + }); + return { rolloutId, revision }; + }); +} + +/** Clone the deployed specification, never mutable service configuration. */ +export async function cloneActiveRevisionAndQueueSystemRollout( + serviceId: string, + sourceRevisionId?: string, +) { + return db.transaction(async (tx) => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${serviceId}))`); + const pending = await tx + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .limit(1) + .then((rows) => rows[0]); + if (pending) return { rolloutId: pending.id, created: false }; + const active = await tx + .select({ specification: serviceRevisions.specification }) + .from(serviceRevisions) + .innerJoin( + deployments, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) + .where( + and( + eq(serviceRevisions.serviceId, serviceId), + ...(sourceRevisionId + ? [eq(serviceRevisions.id, sourceRevisionId)] + : []), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + ), + ) + .orderBy(desc(deployments.createdAt), desc(deployments.id)) + .limit(1) + .then((rows) => rows[0]); + if (!active) throw new Error("Service has no active revision"); + const revisionId = randomUUID(); + await tx.insert(serviceRevisions).values({ + id: revisionId, + serviceId, + specification: active.specification, + actor: { type: "system" }, + }); + const rolloutId = randomUUID(); + await tx.insert(rollouts).values({ + id: rolloutId, + serviceId, + serviceRevisionId: revisionId, + status: "queued", + currentStage: "queued", + }); + return { rolloutId, created: true }; + }); } export async function createRolloutForServiceRevision( @@ -372,6 +427,7 @@ export async function createRolloutForServiceRevision( artifactImageUri: string, ) { return db.transaction(async (tx) => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${serviceId}))`); const [revision, activeService] = await Promise.all([ tx .select() @@ -398,7 +454,10 @@ export async function createRolloutForServiceRevision( if (specification.image !== artifactImageUri) { throw new Error("Built artifact does not match the service revision"); } - if (!specification.placements.some((placement) => placement.count > 0)) { + if ( + specification.placement.mode === "manual" && + !specification.placements.some((placement) => placement.count > 0) + ) { return { rolloutId: null, revision, created: false }; } @@ -441,12 +500,8 @@ export async function getRolloutServiceRevision(rolloutId: string) { .then((rows) => rows[0]); if (!result) throw new Error("Rollout revision not found"); - if ( - result.revision.specification.schemaVersion !== - SERVICE_REVISION_SCHEMA_VERSION - ) { - throw new Error("Unsupported service revision schema version"); - } - - return result.revision; + return { + ...result.revision, + specification: parseServiceRevisionSpec(result.revision.specification), + }; } diff --git a/web/lib/two-factor.ts b/web/lib/two-factor.ts index 2ba72485..eb542749 100644 --- a/web/lib/two-factor.ts +++ b/web/lib/two-factor.ts @@ -4,6 +4,19 @@ export type DeleteConfirmation = { totpCode?: string; }; +export type AuthClientError = { + message?: string; + error_description?: string; +} | null; + +export function getAuthErrorMessage(error: AuthClientError, fallback: string) { + return error?.message || error?.error_description || fallback; +} + +export function normalizeTwoFactorCode(value: string) { + return value.replace(/\s/g, ""); +} + const AUTH_REDIRECT_ORIGIN = "https://auth.local"; function hasUnsafeAuthRedirectCharacter(value: string) { diff --git a/web/lib/victoria-logs.ts b/web/lib/victoria-logs.ts index 075f8fa7..6fd293f3 100644 --- a/web/lib/victoria-logs.ts +++ b/web/lib/victoria-logs.ts @@ -5,41 +5,21 @@ import { normalizeLogCursor, normalizeLogSearch, } from "@/lib/log-query"; +import { + buildFetchOptions, + type EndpointConfig, + parseEndpoint, +} from "@/lib/victoria"; const VICTORIA_LOGS_URL = process.env.VICTORIA_LOGS_URL; const VICTORIA_LOGS_PRIVATE_URL = process.env.VICTORIA_LOGS_PRIVATE_URL; -type EndpointConfig = { - url: string; - username?: string; - password?: string; -}; - -function parseEndpoint(endpoint: string): EndpointConfig { - const parsed = new URL(endpoint); - const username = parsed.username || undefined; - const password = parsed.password || undefined; - parsed.username = ""; - parsed.password = ""; - return { url: parsed.toString().replace(/\/$/, ""), username, password }; -} - function getQueryEndpoint(): EndpointConfig | undefined { const endpoint = VICTORIA_LOGS_PRIVATE_URL || VICTORIA_LOGS_URL; if (!endpoint) return undefined; return parseEndpoint(endpoint); } -function buildFetchOptions(config: EndpointConfig): RequestInit { - if (config.username) { - const credentials = Buffer.from( - `${config.username}:${config.password || ""}`, - ).toString("base64"); - return { headers: { Authorization: `Basic ${credentials}` } }; - } - return {}; -} - export type LogType = "container" | "http"; type LogSearchField = "_msg" | "path" | "method" | "status" | "client_ip"; @@ -162,31 +142,56 @@ function providerSignal(signal?: AbortSignal): AbortSignal { return signal ? AbortSignal.any([signal, timeout]) : timeout; } -async function fetchLogQuery( - endpoint: EndpointConfig, +async function fetchLogQuery( query: string, - signal?: AbortSignal, -): Promise { + { + limit, + pageSize, + errorLabel = "logs", + signal, + serverTimeout = false, + }: { + limit?: number; + pageSize?: number; + errorLabel?: string; + signal?: AbortSignal; + serverTimeout?: boolean; + } = {}, +): Promise<{ logs: T[]; hasMore: boolean }> { + const endpoint = getQueryEndpoint(); + if (!endpoint) { + throw new Error("VICTORIA_LOGS_URL is not configured"); + } + const url = new URL(`${endpoint.url}/select/logsql/query`); url.searchParams.set("query", query); - url.searchParams.set("timeout", "4s"); + const queryLimit = pageSize === undefined ? limit : pageSize + 1; + if (queryLimit !== undefined) { + url.searchParams.set("limit", String(queryLimit)); + } + if (serverTimeout) { + url.searchParams.set("timeout", "4s"); + } const response = await fetch(url.toString(), { ...buildFetchOptions(endpoint), - signal: providerSignal(signal), + signal, }); if (!response.ok) { throw new Error( - `Failed to query logs: ${response.status} ${response.statusText}`, + `Failed to query ${errorLabel}: ${response.status} ${response.statusText}`, ); } const text = await response.text(); - return text + const logs = text .trim() .split("\n") .filter(Boolean) - .map((line) => JSON.parse(line) as StoredLog); + .map((line) => JSON.parse(line) as T); + const hasMore = pageSize !== undefined && logs.length > pageSize; + if (hasMore) logs.pop(); + return { logs, hasMore }; } function deduplicateIdentifiedLogs(logs: StoredLog[]): StoredLog[] { @@ -202,11 +207,6 @@ function deduplicateIdentifiedLogs(logs: StoredLog[]): StoredLog[] { export async function queryPublicServiceLogs( options: PublicServiceLogsOptions, ): Promise<{ logs: StoredLog[]; hasMore: boolean }> { - const endpoint = getQueryEndpoint(); - if (!endpoint) { - throw new Error("VICTORIA_LOGS_URL is not configured"); - } - const pageSize = options.limit + 1; let query = buildServiceLogFilter({ ...options, @@ -232,7 +232,10 @@ export async function queryPublicServiceLogs( query += ` | first ${pageSize} by (_time desc, event_id desc) | sort by (_time, event_id)`; } - const rawLogs = await fetchLogQuery(endpoint, query, options.signal); + const { logs: rawLogs } = await fetchLogQuery(query, { + signal: providerSignal(options.signal), + serverTimeout: true, + }); if ( options.cursor && rawLogs.some((log) => !isPublicServiceLogEventId(log.event_id)) @@ -255,11 +258,6 @@ export async function queryLogsByService( ): Promise<{ logs: StoredLog[]; hasMore: boolean }> { const { limit, after, before } = options; - const endpoint = getQueryEndpoint(); - if (!endpoint) { - throw new Error("VICTORIA_LOGS_URL is not configured"); - } - let query = buildServiceLogFilter(options); const afterCursor = normalizeLogCursor(after); if (afterCursor) { @@ -271,29 +269,10 @@ export async function queryLogsByService( } query += " | sort by (_time desc)"; - const url = new URL(`${endpoint.url}/select/logsql/query`); - url.searchParams.set("query", query); - url.searchParams.set("limit", String(limit + 1)); - - const response = await fetch(url.toString(), { - ...buildFetchOptions(endpoint), + return fetchLogQuery(query, { + pageSize: limit, signal: providerSignal(options.signal), }); - - if (!response.ok) { - throw new Error( - `Failed to query logs: ${response.status} ${response.statusText}`, - ); - } - - const text = await response.text(); - const lines = text.trim().split("\n").filter(Boolean); - const logs = lines.map((line) => JSON.parse(line) as StoredLog); - - const hasMore = logs.length > limit; - if (hasMore) logs.pop(); - - return { logs, hasMore }; } export async function queryLogsByDeployment( @@ -301,11 +280,6 @@ export async function queryLogsByDeployment( limit: number, after?: string, ): Promise<{ logs: StoredLog[]; hasMore: boolean }> { - const endpoint = getQueryEndpoint(); - if (!endpoint) { - throw new Error("VICTORIA_LOGS_URL is not configured"); - } - let query = formatLogSqlExactFilter("deployment_id", deploymentId); const afterCursor = normalizeLogCursor(after); if (afterCursor) { @@ -313,26 +287,7 @@ export async function queryLogsByDeployment( } query += " | sort by (_time desc)"; - const url = new URL(`${endpoint.url}/select/logsql/query`); - url.searchParams.set("query", query); - url.searchParams.set("limit", String(limit + 1)); - - const response = await fetch(url.toString(), buildFetchOptions(endpoint)); - - if (!response.ok) { - throw new Error( - `Failed to query logs: ${response.status} ${response.statusText}`, - ); - } - - const text = await response.text(); - const lines = text.trim().split("\n").filter(Boolean); - const logs = lines.map((line) => JSON.parse(line) as StoredLog); - - const hasMore = logs.length > limit; - if (hasMore) logs.pop(); - - return { logs, hasMore }; + return fetchLogQuery(query, { pageSize: limit }); } export type BuildLog = { @@ -370,11 +325,6 @@ export async function queryLogsByServer({ logs: AgentLog[]; hasMore: boolean; }> { - const endpoint = getQueryEndpoint(); - if (!endpoint) { - throw new Error("VICTORIA_LOGS_URL is not configured"); - } - let query = `${formatLogSqlExactFilter("server_id", serverId)} log_type:agent`; if (range) { query += ` _time:${range}`; @@ -389,26 +339,10 @@ export async function queryLogsByServer({ } query += " | sort by (_time desc)"; - const url = new URL(`${endpoint.url}/select/logsql/query`); - url.searchParams.set("query", query); - url.searchParams.set("limit", String(limit + 1)); - - const response = await fetch(url.toString(), buildFetchOptions(endpoint)); - - if (!response.ok) { - throw new Error( - `Failed to query server logs: ${response.status} ${response.statusText}`, - ); - } - - const text = await response.text(); - const lines = text.trim().split("\n").filter(Boolean); - const logs = lines.map((line) => JSON.parse(line) as AgentLog); - - const hasMore = logs.length > limit; - if (hasMore) logs.pop(); - - return { logs, hasMore }; + return fetchLogQuery(query, { + pageSize: limit, + errorLabel: "server logs", + }); } export type RolloutLog = { @@ -460,11 +394,6 @@ export async function queryLogsByRollout( rolloutId: string, { limit = 1000, search }: { limit?: number; search?: string } = {}, ): Promise<{ logs: RolloutLog[] }> { - const endpoint = getQueryEndpoint(); - if (!endpoint) { - throw new Error("VICTORIA_LOGS_URL is not configured"); - } - let query = `${formatLogSqlExactFilter("rollout_id", rolloutId)} log_type:rollout`; const searchFilter = formatLogSqlSearchFilter(search); if (searchFilter) { @@ -472,21 +401,10 @@ export async function queryLogsByRollout( } query += " | sort by (_time)"; - const url = new URL(`${endpoint.url}/select/logsql/query`); - url.searchParams.set("query", query); - url.searchParams.set("limit", String(limit)); - - const response = await fetch(url.toString(), buildFetchOptions(endpoint)); - - if (!response.ok) { - throw new Error( - `Failed to query rollout logs: ${response.status} ${response.statusText}`, - ); - } - - const text = await response.text(); - const lines = text.trim().split("\n").filter(Boolean); - const logs = lines.map((line) => JSON.parse(line) as RolloutLog); + const { logs } = await fetchLogQuery(query, { + limit, + errorLabel: "rollout logs", + }); return { logs }; } @@ -495,11 +413,6 @@ export async function queryLogsByBuild( buildId: string, { limit = 1000, search }: { limit?: number; search?: string } = {}, ): Promise<{ logs: BuildLog[] }> { - const endpoint = getQueryEndpoint(); - if (!endpoint) { - throw new Error("VICTORIA_LOGS_URL is not configured"); - } - let query = `${formatLogSqlExactFilter("build_id", buildId)} log_type:build`; const searchFilter = formatLogSqlSearchFilter(search); if (searchFilter) { @@ -507,21 +420,10 @@ export async function queryLogsByBuild( } query += " | sort by (_time)"; - const url = new URL(`${endpoint.url}/select/logsql/query`); - url.searchParams.set("query", query); - url.searchParams.set("limit", String(limit)); - - const response = await fetch(url.toString(), buildFetchOptions(endpoint)); - - if (!response.ok) { - throw new Error( - `Failed to query build logs: ${response.status} ${response.statusText}`, - ); - } - - const text = await response.text(); - const lines = text.trim().split("\n").filter(Boolean); - const logs = lines.map((line) => JSON.parse(line) as BuildLog); + const { logs } = await fetchLogQuery(query, { + limit, + errorLabel: "build logs", + }); return { logs }; } diff --git a/web/lib/victoria-metrics.ts b/web/lib/victoria-metrics.ts index d1348567..301b1773 100644 --- a/web/lib/victoria-metrics.ts +++ b/web/lib/victoria-metrics.ts @@ -9,17 +9,16 @@ import { type MetricRange, parseMetricRange, } from "@/lib/metric-ranges"; +import { + buildFetchOptions, + type EndpointConfig, + parseEndpoint, +} from "@/lib/victoria"; export { METRIC_RANGE_OPTIONS, type MetricRange, parseMetricRange }; let hasWarnedMissingMetricsConfig = false; -type EndpointConfig = { - url: string; - username?: string; - password?: string; -}; - type VictoriaInstantResponse = { status: string; data?: { @@ -93,8 +92,6 @@ export type NodeMetricsHistory = { diskUsedBytes: NodeMetricPoint[]; }; -export type MetricsHistory = NodeMetricsHistory; - const METRIC_NAMES = { cpuUsagePercent: "techulus_node_cpu_usage_percent", memoryUsagePercent: "techulus_node_memory_usage_percent", @@ -103,15 +100,6 @@ const METRIC_NAMES = { diskUsedBytes: "techulus_node_disk_used_bytes", } as const; -function parseEndpoint(endpoint: string): EndpointConfig { - const parsed = new URL(endpoint); - const username = parsed.username || undefined; - const password = parsed.password || undefined; - parsed.username = ""; - parsed.password = ""; - return { url: parsed.toString().replace(/\/$/, ""), username, password }; -} - function getQueryEndpoint(): EndpointConfig | undefined { const endpoint = process.env.VICTORIA_METRICS_PRIVATE_URL || @@ -120,16 +108,6 @@ function getQueryEndpoint(): EndpointConfig | undefined { return parseEndpoint(endpoint); } -function buildFetchOptions(config: EndpointConfig): RequestInit { - if (config.username) { - const credentials = Buffer.from( - `${config.username}:${config.password || ""}`, - ).toString("base64"); - return { headers: { Authorization: `Basic ${credentials}` } }; - } - return {}; -} - export function isMetricsEnabled(): boolean { return !!( process.env.VICTORIA_METRICS_PRIVATE_URL || process.env.VICTORIA_METRICS_URL diff --git a/web/lib/victoria.ts b/web/lib/victoria.ts new file mode 100644 index 00000000..1bf91a0d --- /dev/null +++ b/web/lib/victoria.ts @@ -0,0 +1,24 @@ +export type EndpointConfig = { + url: string; + username?: string; + password?: string; +}; + +export function parseEndpoint(endpoint: string): EndpointConfig { + const parsed = new URL(endpoint); + const username = parsed.username || undefined; + const password = parsed.password || undefined; + parsed.username = ""; + parsed.password = ""; + return { url: parsed.toString().replace(/\/$/, ""), username, password }; +} + +export function buildFetchOptions(config: EndpointConfig): RequestInit { + if (config.username) { + const credentials = Buffer.from( + `${config.username}:${config.password || ""}`, + ).toString("base64"); + return { headers: { Authorization: `Basic ${credentials}` } }; + } + return {}; +} diff --git a/web/lib/work-queue.ts b/web/lib/work-queue.ts index 8379998c..73560bd7 100644 --- a/web/lib/work-queue.ts +++ b/web/lib/work-queue.ts @@ -10,6 +10,65 @@ import { inngestEvents } from "@/lib/inngest/events"; export const WORK_QUEUE_MAX_ATTEMPTS = 3; export const WORK_QUEUE_LEASE_DURATION_MS = 2 * MINUTE_IN_MILLISECONDS; +type WorkQueueStorageConfig = { + provider: string; + bucket: string; + region: string; + endpoint: string; + accessKey: string; + secretKey: string; +}; + +type ReconcileWorkPayload = { + reason: string; + deploymentId?: string; +}; + +export type WorkPayloadByType = { + deploy: ReconcileWorkPayload; + reconcile: ReconcileWorkPayload; + stop: { deploymentId: string; containerId: string | null }; + restart: { + deploymentId: string; + containerId: string | null; + reason?: string; + }; + force_cleanup: { + serviceId: string; + containerIds: string[]; + reason?: string; + deploymentId?: string; + }; + cleanup_volumes: { serviceId: string }; + build: { buildId: string }; + backup_volume: { + backupId: string; + serviceId: string; + containerId: string | null; + volumeName: string; + storagePath: string; + storageConfig: WorkQueueStorageConfig; + }; + restore_volume: { + backupId: string; + serviceId: string; + containerId?: string | null; + volumeName: string; + storagePath: string; + expectedChecksum: string; + isMigrationRestore: boolean; + storageConfig: WorkQueueStorageConfig; + }; + create_manifest: { + images: string[]; + finalImageUri: string; + serviceId: string; + serviceRevisionId: string; + buildGroupId: string; + }; + upgrade_agent: { targetVersion: string; expectedSha256: string }; +}; + export type WorkItemResult = { id: string; attempt: number; @@ -39,10 +98,10 @@ export type RejectedActiveWorkItem = { reason: string; }; -export async function enqueueWork( +export async function enqueueWork( serverId: string, - type: WorkQueue["type"], - payload: Record, + type: T, + payload: WorkPayloadByType[T], options: { id?: string } = {}, ) { await db @@ -249,12 +308,9 @@ async function runWorkItemCompletionSideEffects( } try { - const payload = JSON.parse(item.payload) as { - serviceId?: string; - serviceRevisionId?: string; - finalImageUri?: string; - buildGroupId?: string; - }; + const payload = JSON.parse(item.payload) as Partial< + WorkPayloadByType["create_manifest"] + >; if (result.status === "completed") { if ( diff --git a/web/package.json b/web/package.json index 87967c96..b86ac9b2 100644 --- a/web/package.json +++ b/web/package.json @@ -67,7 +67,6 @@ "eslint": "^9", "eslint-config-next": "16.2.9", "tailwindcss": "^4", - "tsx": "^4.19.2", "typescript": "^5", "vitest": "^4.1.9" }, diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index dc11523b..022ae4e6 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -168,9 +168,6 @@ importers: tailwindcss: specifier: ^4 version: 4.3.1 - tsx: - specifier: ^4.19.2 - version: 4.22.4 typescript: specifier: ^5 version: 5.9.3 diff --git a/web/public/update.sh b/web/public/update.sh deleted file mode 100644 index c9a454b1..00000000 --- a/web/public/update.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -set -e - -error() { - echo "" - echo "ERROR: $1" >&2 - exit 1 -} - -echo "" -echo "==========================================" -echo " Techulus Cloud Agent Updater" -echo "==========================================" -echo "" - -if [ "$(id -u)" -ne 0 ]; then - error "This script must be run as root" -fi - -if [ ! -f /usr/local/bin/techulus-agent ]; then - error "Agent not installed. Run setup.sh first." -fi - -ARCH=$(uname -m) -case $ARCH in - x86_64) - AGENT_ARCH="amd64" - ;; - aarch64) - AGENT_ARCH="arm64" - ;; - *) - error "Unsupported architecture: $ARCH" - ;; -esac -echo "Architecture: $ARCH ($AGENT_ARCH)" - -LATEST_VERSION=$(curl -fsSL "https://api.github.com/repos/techulus/cloud/releases/latest" | sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p') -if [ -z "$LATEST_VERSION" ]; then - error "Failed to resolve latest version" -fi -echo "Updating to version: $LATEST_VERSION" - -AGENT_URL="https://github.com/techulus/cloud/releases/latest/download/agent-linux-${AGENT_ARCH}" -CHECKSUM_URL="https://github.com/techulus/cloud/releases/latest/download/checksums.txt" - -curl -fsSL -o /tmp/techulus-agent "$AGENT_URL" -if [ ! -f /tmp/techulus-agent ]; then - error "Failed to download agent binary" -fi - -curl -fsSL -o /tmp/checksums.txt "$CHECKSUM_URL" -if [ ! -f /tmp/checksums.txt ]; then - rm -f /tmp/techulus-agent - error "Failed to download checksums file" -fi - -EXPECTED_CHECKSUM=$(grep "agent-linux-${AGENT_ARCH}" /tmp/checksums.txt | awk '{print $1}') -if [ -z "$EXPECTED_CHECKSUM" ]; then - rm -f /tmp/techulus-agent /tmp/checksums.txt - error "Could not find checksum for agent-linux-${AGENT_ARCH}" -fi - -ACTUAL_CHECKSUM=$(sha256sum /tmp/techulus-agent | awk '{print $1}') -if [ "$EXPECTED_CHECKSUM" != "$ACTUAL_CHECKSUM" ]; then - rm -f /tmp/techulus-agent /tmp/checksums.txt - error "Checksum verification failed" -fi -echo "✓ Checksum verified" -rm -f /tmp/checksums.txt - -chmod +x /tmp/techulus-agent -mv /tmp/techulus-agent /usr/local/bin/techulus-agent - -echo "Restarting agent..." -systemctl restart techulus-agent -sleep 3 - -if ! systemctl is-active --quiet techulus-agent; then - journalctl -u techulus-agent --no-pager -n 20 - error "Failed to start agent" -fi - -echo "" -echo "==========================================" -echo " Update completed successfully!" -echo "==========================================" -echo "" -echo "Agent status: $(systemctl is-active techulus-agent)" -echo "" diff --git a/web/tests/agent-status.test.ts b/web/tests/agent-status.test.ts index a3fc860c..e7553218 100644 --- a/web/tests/agent-status.test.ts +++ b/web/tests/agent-status.test.ts @@ -49,10 +49,11 @@ vi.mock("@/lib/work-queue", () => ({ import { applyStatusReport, getSleepTransitionDeploymentIds, - getStaleStoppedServerlessReportUpdate, + getStaleStoppedReportUpdate, getStoppedContainerReportUpdate, shouldAttachReportedContainer, } from "@/lib/agent-status"; +import { inngest } from "@/lib/inngest/client"; beforeEach(() => { mocks.selectResults.length = 0; @@ -97,9 +98,9 @@ describe("agent status serverless attachment", () => { }); }); - it("restores stale stopped serverless observations from live running reports", () => { + it("restores stale stopped observations from live running reports", () => { expect( - getStaleStoppedServerlessReportUpdate({ + getStaleStoppedReportUpdate({ hasHealthCheck: false, healthStatus: "none", }), @@ -110,7 +111,7 @@ describe("agent status serverless attachment", () => { }); expect( - getStaleStoppedServerlessReportUpdate({ + getStaleStoppedReportUpdate({ hasHealthCheck: true, healthStatus: "starting", }), @@ -121,7 +122,7 @@ describe("agent status serverless attachment", () => { }); expect( - getStaleStoppedServerlessReportUpdate({ + getStaleStoppedReportUpdate({ hasHealthCheck: true, healthStatus: "healthy", }), @@ -168,6 +169,34 @@ describe("agent status deployment cleanup", () => { expect(mocks.db.delete).toHaveBeenCalledTimes(1); }); + it("retains a removed deployment whose container is reported in a transient state", async () => { + const deployment = { + id: "deployment_removed", + serviceId: "service_1", + serviceRevisionId: "revision_1", + serverId: "server_1", + containerId: "container_1", + runtimeDesiredState: "removed", + trafficState: "inactive", + observedPhase: "sleeping", + rolloutId: "rollout_1", + }; + mocks.selectResults.push([deployment]); + + await applyStatusReport("server_1", { + containers: [ + { + deploymentId: deployment.id, + containerId: "container_1", + status: "transient", + healthStatus: "none", + }, + ], + }); + + expect(mocks.db.delete).not.toHaveBeenCalled(); + }); + it("retains a removed containerless deployment that reappears in the report", async () => { const deployment = { id: "deployment_removed", @@ -196,3 +225,83 @@ describe("agent status deployment cleanup", () => { expect(mocks.db.delete).not.toHaveBeenCalled(); }); }); + +describe("agent status stopped-phase recovery", () => { + it("promotes a non-serverless stopped deployment with a running container and notifies its rollout", async () => { + const deployment = { + id: "deployment_1", + serviceId: "service_1", + serviceRevisionId: "revision_1", + serverId: "server_1", + containerId: "container_1", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "stopped", + rolloutId: "rollout_1", + }; + mocks.selectResults.push( + [deployment], + [deployment], + [ + { + specification: { serverless: { enabled: false }, healthCheck: null }, + }, + ], + ); + + await applyStatusReport("server_1", { + containers: [ + { + deploymentId: deployment.id, + containerId: "container_1", + status: "running", + healthStatus: "none", + }, + ], + }); + + expect(inngest.send).toHaveBeenCalledWith( + expect.objectContaining({ + type: "deployment", + id: "deployment_1", + parentType: "rollout", + parentId: "rollout_1", + }), + ); + }); + + it("notifies the rollout when an unknown deployment is restored to running", async () => { + const deployment = { + id: "deployment_unknown", + serviceId: "service_1", + serviceRevisionId: "revision_1", + serverId: "server_1", + containerId: "container_1", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "unknown", + rolloutId: "rollout_2", + }; + mocks.selectResults.push([deployment], [deployment]); + + await applyStatusReport("server_1", { + containers: [ + { + deploymentId: deployment.id, + containerId: "container_1", + status: "running", + healthStatus: "none", + }, + ], + }); + + expect(inngest.send).toHaveBeenCalledWith( + expect.objectContaining({ + type: "deployment", + id: "deployment_unknown", + parentType: "rollout", + parentId: "rollout_2", + }), + ); + }); +}); diff --git a/web/tests/autoplacement.test.ts b/web/tests/autoplacement.test.ts new file mode 100644 index 00000000..eae08e90 --- /dev/null +++ b/web/tests/autoplacement.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import { + automaticPlacementIneligibilityReason, + distributeReplicas, +} from "@/lib/inngest/functions/rollout-helpers"; +import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import { getServiceRevisionTotalReplicas } from "@/lib/service-revision-spec"; + +describe("automatic placement distribution", () => { + it("uses placement intent to determine a revision replica total", () => { + expect( + getServiceRevisionTotalReplicas({ + placement: { mode: "automatic", replicas: 4 }, + placements: [], + }), + ).toBe(4); + expect( + getServiceRevisionTotalReplicas({ + placement: { mode: "manual" }, + placements: [ + { serverId: "a", count: 1 }, + { serverId: "b", count: 2 }, + ], + }), + ).toBe(3); + }); + + it.each([ + [1, [1, 0, 0]], + [3, [1, 1, 1]], + [5, [2, 2, 1]], + ])("distributes %i replicas over three servers", (replicas, counts) => { + const result = distributeReplicas(["c", "a", "b"], replicas); + expect(result.map((placement) => placement.serverId)).toEqual( + ["a", "b", "c"].slice(0, result.length), + ); + expect( + ["a", "b", "c"].map( + (id) => result.find((p) => p.serverId === id)?.replicas ?? 0, + ), + ).toEqual(counts); + }); + + it("stacks ten replicas deterministically on two servers", () => { + expect(distributeReplicas(["b", "a"], 10)).toEqual([ + { serverId: "a", replicas: 5 }, + { serverId: "b", replicas: 5 }, + ]); + }); +}); + +describe("automatic placement eligibility diagnostics", () => { + const eligible = { + status: "online", + wireguardIp: "10.0.0.2", + isProxy: false, + }; + + it.each([ + [{ ...eligible, status: "offline" }, "status is offline"], + [{ ...eligible, wireguardIp: null }, "WireGuard is not configured"], + ])("reports why a server is ineligible", (server, reason) => { + expect(automaticPlacementIneligibilityReason(server)).toBe(reason); + }); + + it("reports proxy eligibility only when required", () => { + expect(automaticPlacementIneligibilityReason(eligible)).toBeNull(); + expect(automaticPlacementIneligibilityReason(eligible, true)).toBe( + "not a proxy node", + ); + }); +}); + +describe("persisted revision compatibility", () => { + it("normalizes v2 revisions to v3 manual intent", () => { + const parsed = parseServiceRevisionSpec({ + schemaVersion: 2, + image: "nginx", + source: { type: "image", image: "nginx" }, + hostname: "web", + stateful: false, + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + healthCheck: null, + startCommand: null, + resourceLimits: { cpuCores: null, memoryMb: null }, + placements: [{ serverId: "a", count: 1 }], + ports: [], + secrets: [], + volumes: [], + }); + expect(parsed.schemaVersion).toBe(3); + expect(parsed.placement).toEqual({ mode: "manual" }); + }); +}); diff --git a/web/tests/build-actions.test.ts b/web/tests/build-actions.test.ts index 04b86174..7d50e752 100644 --- a/web/tests/build-actions.test.ts +++ b/web/tests/build-actions.test.ts @@ -5,7 +5,6 @@ const mocks = vi.hoisted(() => { service: { id: "service-1", sourceType: "github", - githubRootDir: "apps/web", }, githubRepo: { id: "repo-link-1", @@ -31,8 +30,6 @@ const mocks = vi.hoisted(() => { db: { select: vi.fn(() => query) }, requireDeveloperRole: vi.fn(), listGitHubCommits: vi.fn(), - send: vi.fn(), - createBuildTrigger: vi.fn((data) => ({ name: "build/trigger", data })), triggerResolvedBuildInternal: vi.fn(), }; }); @@ -46,16 +43,14 @@ vi.mock("@/lib/github", async (importOriginal) => ({ listGitHubCommits: mocks.listGitHubCommits, })); vi.mock("@/lib/inngest/client", () => ({ - inngest: { send: mocks.send }, + inngest: { send: vi.fn() }, })); vi.mock("@/lib/inngest/events", () => ({ inngestEvents: { - buildTrigger: { create: mocks.createBuildTrigger }, + buildTrigger: { create: vi.fn() }, }, })); vi.mock("@/lib/trigger-build", () => ({ - triggerBuildInternal: vi.fn(), - requeueBuildRevisionInternal: vi.fn(), triggerResolvedBuildInternal: mocks.triggerResolvedBuildInternal, })); @@ -65,7 +60,6 @@ const selectedCommit = { sha: "0123456789abcdef0123456789abcdef01234567", message: "Deploy this commit", author: "octocat", - date: "2026-07-18T00:00:00Z", }; describe("manual commit builds", () => { @@ -76,8 +70,6 @@ describe("manual commit builds", () => { user: { id: "user-1", name: "Alice" }, }); mocks.listGitHubCommits.mockReset(); - mocks.send.mockReset(); - mocks.createBuildTrigger.mockClear(); mocks.triggerResolvedBuildInternal.mockReset(); mocks.triggerResolvedBuildInternal.mockResolvedValue({ status: "queued" }); }); diff --git a/web/tests/build-assignment.test.ts b/web/tests/build-assignment.test.ts index 7a18320f..7e16602a 100644 --- a/web/tests/build-assignment.test.ts +++ b/web/tests/build-assignment.test.ts @@ -39,7 +39,8 @@ function specification( overrides: Partial = {}, ): ServiceRevisionSpec { return { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image: "registry/app:revision-1", source: { type: "image", image: "registry/app:revision-1" }, hostname: "app", @@ -154,6 +155,14 @@ describe("revision-backed build assignment", () => { ).resolves.toBe("server-arm"); }); + it("falls back to cross-platform builds when no native builder is online", async () => { + mocks.getSetting.mockResolvedValue(null); + mocks.queryResults.push([{ id: "server-arm", meta: { arch: "arm64" } }]); + await expect( + selectBuildServerForRevision(specification(), "linux/amd64"), + ).resolves.toBe("server-arm"); + }); + it("limits stateless build assignment to configured build servers", async () => { mocks.getSetting.mockResolvedValue(["server-arm"]); mocks.queryResults.push([{ id: "server-arm", meta: { arch: "arm64" } }]); diff --git a/web/tests/deploy-service-revision.test.ts b/web/tests/deploy-service-revision.test.ts index ef044af3..f13b2505 100644 --- a/web/tests/deploy-service-revision.test.ts +++ b/web/tests/deploy-service-revision.test.ts @@ -8,6 +8,7 @@ const mocks = vi.hoisted(() => ({ startMigrationInternal: vi.fn(), triggerBuildInternal: vi.fn(), send: vi.fn(), + updateWhere: vi.fn(), createRolloutCreated: vi.fn((data, options) => ({ name: "rollout/created", data, @@ -17,6 +18,9 @@ const mocks = vi.hoisted(() => ({ vi.mock("@/db", () => ({ db: { + update: vi.fn(() => ({ + set: vi.fn(() => ({ where: mocks.updateWhere })), + })), select: vi.fn(() => ({ from: vi.fn(() => ({ where: vi.fn(() => Promise.resolve(mocks.replicaRows)), @@ -129,6 +133,7 @@ describe("revision rollout idempotency", () => { expect(mocks.send.mock.calls[0]?.[0].id).toBe( mocks.send.mock.calls[1]?.[0].id, ); + expect(mocks.updateWhere).toHaveBeenCalledTimes(1); }); it("bases a GitHub redeployment on an explicit runtime revision", async () => { diff --git a/web/tests/expected-state.test.ts b/web/tests/expected-state.test.ts index 864dda64..138355a0 100644 --- a/web/tests/expected-state.test.ts +++ b/web/tests/expected-state.test.ts @@ -527,16 +527,6 @@ describe("expected-state pure builders", () => { }, }), ]; - const ports: Parameters[0]["ports"] = [ - { - id: "port_1", - serviceId: "svc_serverless", - port: 3000, - isPublic: true, - protocol: "http", - domain: "sleepy.example.com", - }, - ] as Parameters[0]["ports"]; const { serverlessServiceIds, serverlessRouteSuppressedServiceIds } = buildServerlessTraefikRouteSets({ @@ -549,43 +539,6 @@ describe("expected-state pure builders", () => { expect(Array.from(serverlessServiceIds)).toEqual(["svc_serverless"]); expect(Array.from(serverlessRouteSuppressedServiceIds)).toEqual([]); - - const routes = buildTraefikRoutes({ - serverId: "proxy_1", - ports, - routableDeployments: [], - serverlessServiceIds, - serverlessRouteSuppressedServiceIds, - }); - - expect(routes.httpRoutes).toEqual([ - { - id: "sleepy.example.com", - domain: "sleepy.example.com", - serviceId: "svc_serverless", - upstreams: [{ url: "127.0.0.1:18080", weight: 1 }], - }, - ]); - }); - - it("omits serverless HTTP routes on non-owner proxies", () => { - const routes = buildTraefikRoutes({ - serverId: "proxy_2", - ports: [ - { - id: "port_1", - serviceId: "svc_serverless", - port: 3000, - isPublic: true, - protocol: "http", - domain: "sleepy.example.com", - }, - ] as any, - routableDeployments: [], - serverlessRouteSuppressedServiceIds: new Set(["svc_serverless"]), - }); - - expect(routes.httpRoutes).toEqual([]); }); it("keeps suppressed serverless HTTP domains in certificate selection", () => { diff --git a/web/tests/log-routes.test.ts b/web/tests/log-routes.test.ts index b8e9261c..81765679 100644 --- a/web/tests/log-routes.test.ts +++ b/web/tests/log-routes.test.ts @@ -5,7 +5,6 @@ describe("log routes", () => { vi.restoreAllMocks(); vi.resetModules(); vi.doUnmock("next/headers"); - vi.doUnmock("@/lib/api-auth"); vi.doUnmock("@/lib/auth"); vi.doUnmock("@/lib/victoria-logs"); }); @@ -121,9 +120,7 @@ describe("log routes", () => { }); }); -async function loadServiceLogsRoute( - queryLogsByService = vi.fn(async () => ({ logs: [], hasMore: false })), -) { +function mockAuthenticatedSession() { vi.resetModules(); vi.doMock("next/headers", () => ({ headers: async () => new Headers(), @@ -135,6 +132,12 @@ async function loadServiceLogsRoute( }, }, })); +} + +async function loadServiceLogsRoute( + queryLogsByService = vi.fn(async () => ({ logs: [], hasMore: false })), +) { + mockAuthenticatedSession(); vi.doMock("@/lib/victoria-logs", () => ({ isLoggingEnabled: () => true, queryLogsByService, @@ -147,17 +150,7 @@ async function loadServiceLogsRoute( async function loadDeploymentLogsRoute( queryLogsByDeployment = vi.fn(async () => ({ logs: [], hasMore: false })), ) { - vi.resetModules(); - vi.doMock("next/headers", () => ({ - headers: async () => new Headers(), - })); - vi.doMock("@/lib/auth", () => ({ - auth: { - api: { - getSession: async () => ({ user: { id: "user-1" } }), - }, - }, - })); + mockAuthenticatedSession(); vi.doMock("@/lib/victoria-logs", () => ({ isLoggingEnabled: () => true, queryLogsByDeployment, diff --git a/web/tests/project-deletion.test.ts b/web/tests/project-deletion.test.ts deleted file mode 100644 index 519ff95c..00000000 --- a/web/tests/project-deletion.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { blocksProjectDeletion } from "@/lib/project-deletion"; - -describe("project deletion guard", () => { - it("blocks deletion for deployments that still have runtime intent", () => { - expect(blocksProjectDeletion({ runtimeDesiredState: "running" })).toBe(true); - expect(blocksProjectDeletion({ runtimeDesiredState: "stopped" })).toBe(true); - expect(blocksProjectDeletion({ runtimeDesiredState: "removed" })).toBe( - false, - ); - }); -}); diff --git a/web/tests/public-api-configuration.test.ts b/web/tests/public-api-configuration.test.ts index a255c6f6..f17dce62 100644 --- a/web/tests/public-api-configuration.test.ts +++ b/web/tests/public-api-configuration.test.ts @@ -89,4 +89,35 @@ describe("public API configuration state", () => { expect(configuration.hasPendingChanges).toBe(false); expect(configuration.changes).toEqual([]); }); + + it("serializes automatic placement from desired service replicas", async () => { + mocks.rows.push([], [], [], [], []); + const configuration = await safeConfiguration({ + id: "service-1", + name: "Automatic", + sourceType: "image", + image: "nginx", + hostname: null, + stateful: false, + placementMode: "automatic", + replicas: 4, + healthCheckCmd: null, + startCommand: null, + resourceCpuLimit: null, + resourceMemoryLimitMb: null, + serverlessEnabled: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 300, + deploymentSchedule: null, + backupEnabled: false, + backupSchedule: null, + } as never); + + expect(configuration.current.placement).toEqual({ + mode: "automatic", + replicas: 4, + }); + expect(configuration.current.replicas).toBe(4); + expect(configuration.current.placements).toEqual([]); + }); }); diff --git a/web/tests/public-api-revisions-auth.test.ts b/web/tests/public-api-revisions-auth.test.ts index 57daeac2..b6a61597 100644 --- a/web/tests/public-api-revisions-auth.test.ts +++ b/web/tests/public-api-revisions-auth.test.ts @@ -55,7 +55,6 @@ const mocks = vi.hoisted(() => ({ vi.mock("@/lib/api-auth", () => ({ requireApiKeyRole: mocks.requireApiKeyRole, - requireApiKeyDeveloperRole: vi.fn(), requireRequestSession: mocks.requireRequestSession, })); diff --git a/web/tests/public-api-source.test.ts b/web/tests/public-api-source.test.ts index a31698e4..030625a6 100644 --- a/web/tests/public-api-source.test.ts +++ b/web/tests/public-api-source.test.ts @@ -113,3 +113,43 @@ describe("public API GitHub sources", () => { expect(cleared).toHaveProperty("rootDir", null); }); }); + +describe("public API placement schema", () => { + it("rejects the removed standalone replicas field", () => { + expect(configurationPatchSchema.safeParse({ replicas: 3 }).success).toBe( + false, + ); + }); + + it.each([ + { mode: "automatic", replicas: 3 }, + { mode: "manual", placements: [{ serverId: "server-1", count: 2 }] }, + ])("accepts valid placement intent", (placement) => { + expect(configurationPatchSchema.safeParse({ placement }).success).toBe( + true, + ); + }); + + it.each([ + { mode: "automatic", replicas: 0 }, + { mode: "manual", placements: [] }, + { + mode: "manual", + placements: [ + { serverId: "a", count: 6 }, + { serverId: "b", count: 5 }, + ], + }, + { + mode: "manual", + placements: [ + { serverId: "a", count: 1 }, + { serverId: "a", count: 1 }, + ], + }, + ])("rejects invalid placement intent", (placement) => { + expect(configurationPatchSchema.safeParse({ placement }).success).toBe( + false, + ); + }); +}); diff --git a/web/tests/rollout-turn.test.ts b/web/tests/rollout-turn.test.ts new file mode 100644 index 00000000..fc4f1b10 --- /dev/null +++ b/web/tests/rollout-turn.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + selectResults: [] as unknown[][], + set: vi.fn(), +})); + +vi.mock("@/db", () => ({ + db: { + transaction: vi.fn(async (callback) => { + const tx = { + execute: vi.fn(), + select: vi.fn(() => { + const isInitialRolloutQuery = mocks.selectResults.length === 2; + const rows = mocks.selectResults.shift() ?? []; + return { + from: () => ({ + where: () => + isInitialRolloutQuery + ? Promise.resolve(rows) + : { limit: () => Promise.resolve(rows) }, + }), + }; + }), + update: vi.fn(() => ({ + set: (value: unknown) => { + mocks.set(value); + return { where: vi.fn() }; + }, + })), + }; + return callback(tx); + }), + }, +})); + +vi.mock("@/lib/inngest/client", () => ({ + inngest: { createFunction: vi.fn(() => ({})) }, +})); +vi.mock("@/lib/inngest/events", () => ({ + inngestEvents: { + rolloutCreated: {}, + rolloutCancelled: {}, + }, +})); + +import { acquireRolloutTurn } from "@/lib/inngest/functions/rollout-workflow"; + +describe("rollout turn acquisition", () => { + it("supersedes a delayed enqueue failure when another intent exists", async () => { + mocks.selectResults.push( + [ + { + status: "failed", + currentStage: "enqueue_failed", + createdAt: new Date("2026-07-23T10:00:00Z"), + }, + ], + [{ id: "newer-rollout" }], + ); + + await expect( + acquireRolloutTurn("delayed-rollout", "service-1"), + ).resolves.toBe("terminal"); + expect(mocks.set).toHaveBeenCalledWith({ currentStage: "superseded" }); + expect(mocks.set).not.toHaveBeenCalledWith( + expect.objectContaining({ status: "in_progress" }), + ); + }); +}); diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts index f6f65794..95933302 100644 --- a/web/tests/service-config.test.ts +++ b/web/tests/service-config.test.ts @@ -4,6 +4,7 @@ import { type DeployedConfig, diffConfigs, getCurrentServerlessConfig, + getServiceTotalReplicas, hasBuildAffectingChanges, MIN_SERVERLESS_SLEEP_AFTER_SECONDS, revisionSpecToDeployedConfig, @@ -29,6 +30,20 @@ function deployedConfig( } describe("service config", () => { + it("uses the placement mode to determine the configured replica total", () => { + const service = { + replicas: 4, + configuredReplicas: [{ count: 1 }, { count: 2 }], + }; + + expect( + getServiceTotalReplicas({ ...service, placementMode: "automatic" }), + ).toBe(4); + expect( + getServiceTotalReplicas({ ...service, placementMode: "manual" }), + ).toBe(3); + }); + it("enforces the minimum serverless sleep timeout", () => { expect( getCurrentServerlessConfig({ @@ -44,7 +59,8 @@ describe("service config", () => { it("converts an immutable revision for pending-change comparisons", () => { const config = revisionSpecToDeployedConfig( { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image: "nginx", source: { type: "image", image: "nginx" }, hostname: "api", @@ -114,7 +130,8 @@ describe("service config", () => { ] as const) { const active = revisionSpecToDeployedConfig( { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image, source: { ...currentSource, diff --git a/web/tests/service-revision-build.test.ts b/web/tests/service-revision-build.test.ts index 3539c9d5..b038673a 100644 --- a/web/tests/service-revision-build.test.ts +++ b/web/tests/service-revision-build.test.ts @@ -35,6 +35,7 @@ const mocks = vi.hoisted(() => { return query; } const tx = { + execute: vi.fn(() => Promise.resolve()), select: vi.fn(() => selectQuery(selectResults.shift() ?? [])), insert: vi.fn(() => insertQuery()), }; @@ -64,7 +65,8 @@ import { function sourceSpecification(): ServiceRevisionSpec { return { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image: "registry.test/project-1/service-1:revision-original", source: { type: "github", @@ -162,6 +164,7 @@ describe("GitHub build service revisions", () => { actor: { type: "system" }, }), ).rejects.toThrow("Service revision idempotency conflict"); + expect(mocks.tx.execute).toHaveBeenCalledTimes(1); expect(mocks.tx.insert).not.toHaveBeenCalled(); }); @@ -202,6 +205,7 @@ describe("GitHub build service revisions", () => { { type: "system" }, "revision-active", ); + expect(mocks.db.transaction.mock.calls.at(-1)).toHaveLength(1); const inserted = mocks.insertedValues[0] as { specification: ServiceRevisionSpec; diff --git a/web/tests/service-revision-changes.test.ts b/web/tests/service-revision-changes.test.ts index 84dd722c..6304b160 100644 --- a/web/tests/service-revision-changes.test.ts +++ b/web/tests/service-revision-changes.test.ts @@ -1,10 +1,14 @@ import { describe, expect, it } from "vitest"; -import { diffServiceRevisionSpecs } from "@/lib/service-revision-changes"; +import { + diffServiceRevisionSpecs, + parseServiceRevisionSpec, +} from "@/lib/service-revision-changes"; import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; function spec(): ServiceRevisionSpec { return { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image: "app:v1", source: { type: "image", image: "app:v1" }, hostname: "app", @@ -46,20 +50,8 @@ describe("diffServiceRevisionSpecs", () => { const previous = spec(); const current = structuredClone(previous); current.image = "app:v2"; - current.hostname = "new-app"; - current.stateful = true; - current.serverless = { - enabled: true, - sleepAfterSeconds: 600, - wakeTimeoutSeconds: 90, - }; - current.healthCheck = { - cmd: "wget /ready", - interval: 20, - timeout: 8, - retries: 5, - startPeriod: 40, - }; + if (!current.healthCheck) throw new Error("Expected health check fixture"); + current.healthCheck.startPeriod = 40; current.startCommand = "npm start"; current.resourceLimits = { cpuCores: 2, memoryMb: 512 }; @@ -138,6 +130,34 @@ describe("diffServiceRevisionSpecs", () => { }); }); + it("reports automatic placement intent without transient server assignments", () => { + const previous = spec(); + previous.placement = { mode: "automatic", replicas: 2 }; + previous.placements = []; + const current = structuredClone(previous); + current.placement = { mode: "automatic", replicas: 4 }; + + expect(diffServiceRevisionSpecs(previous, current)).toEqual([ + { field: "Desired replicas", from: "2", to: "4" }, + ]); + + const manual = spec(); + expect(diffServiceRevisionSpecs(manual, previous)).toEqual([ + { field: "Placement mode", from: "Manual", to: "Automatic" }, + ]); + }); + + it("rejects persisted automatic serverless revisions", () => { + const automatic = spec(); + automatic.placement = { mode: "automatic", replicas: 1 }; + automatic.placements = []; + automatic.serverless.enabled = true; + + expect(() => parseServiceRevisionSpec(automatic)).toThrow( + "Serverless services cannot use automatic placement", + ); + }); + it("never exposes secret ciphertext while detecting additions, updates, and removals", () => { const previous = spec(); previous.secrets.push({ diff --git a/web/tests/service-revision-spec.test.ts b/web/tests/service-revision-spec.test.ts index 4fd3add4..1bf4831b 100644 --- a/web/tests/service-revision-spec.test.ts +++ b/web/tests/service-revision-spec.test.ts @@ -120,7 +120,7 @@ describe("service revision specification", () => { }); expect(spec).toMatchObject({ - schemaVersion: 2, + schemaVersion: 3, image: "registry.test/project/service:revision-1", source: { type: "github", @@ -141,6 +141,45 @@ describe("service revision specification", () => { ).not.toThrow(); }); + it("snapshots automatic placement intent without resolved placements", () => { + const input = draft({ volumes: [] }); + input.service.placementMode = "automatic"; + input.service.replicas = 4; + + expect(buildServiceRevisionSpec(input)).toMatchObject({ + placement: { mode: "automatic", replicas: 4 }, + placements: [], + }); + }); + + it("rejects automatic placement for stateful, volume-backed, and serverless services", () => { + const stateful = draft({ volumes: [] }); + stateful.service.stateful = true; + stateful.service.placementMode = "automatic"; + stateful.service.replicas = 1; + + expect(() => buildServiceRevisionSpec(stateful)).toThrow( + "Stateful services cannot use automatic placement", + ); + + const volumeBacked = draft(); + volumeBacked.service.placementMode = "automatic"; + volumeBacked.service.replicas = 1; + + expect(() => buildServiceRevisionSpec(volumeBacked)).toThrow( + "Services with volumes cannot use automatic placement", + ); + + const serverless = draft({ volumes: [] }); + serverless.service.serverlessEnabled = true; + serverless.service.placementMode = "automatic"; + serverless.service.replicas = 1; + + expect(() => buildServiceRevisionSpec(serverless)).toThrow( + "Serverless services cannot use automatic placement", + ); + }); + it("rejects serverless revisions without a public HTTP port and domain", () => { const input = draft({ ports: [ diff --git a/web/tests/service-revisions-route.test.ts b/web/tests/service-revisions-route.test.ts index 43f5955e..673da3f0 100644 --- a/web/tests/service-revisions-route.test.ts +++ b/web/tests/service-revisions-route.test.ts @@ -40,7 +40,8 @@ function revisionSpec( encryptedValue = "cipher", ): ServiceRevisionSpec { return { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image, source: { type: "image", image }, hostname: "app", diff --git a/web/tests/trigger-build.test.ts b/web/tests/trigger-build.test.ts index bd627261..7fc82c48 100644 --- a/web/tests/trigger-build.test.ts +++ b/web/tests/trigger-build.test.ts @@ -5,7 +5,7 @@ const mocks = vi.hoisted(() => ({ select: vi.fn(), send: vi.fn(), resolveGitHubCommit: vi.fn(), - createBuildTrigger: vi.fn((data) => ({ name: "build/trigger", data })), + createBuildTrigger: vi.fn(), createGitHubBuildServiceRevision: vi.fn(), cloneGitHubBuildServiceRevision: vi.fn(), })); diff --git a/web/tests/victoria-logs.test.ts b/web/tests/victoria-logs.test.ts index d1c40861..4e2b62f3 100644 --- a/web/tests/victoria-logs.test.ts +++ b/web/tests/victoria-logs.test.ts @@ -13,7 +13,6 @@ import { describe("VictoriaLogs queries", () => { afterEach(() => { - vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.resetModules(); @@ -114,10 +113,12 @@ describe("VictoriaLogs queries", () => { it("searches service logs before applying the result limit and range", async () => { const { queryLogsByService } = await loadVictoriaLogs(); const urls: URL[] = []; + let requestSignal: AbortSignal | null | undefined; vi.stubGlobal( "fetch", - vi.fn(async (input: string | URL | Request) => { + vi.fn(async (input: string | URL | Request, init?: RequestInit) => { urls.push(new URL(String(input))); + requestSignal = init?.signal; return jsonLinesResponse([ storedLog("2026-07-10T01:00:00Z", "first match"), storedLog("2026-07-10T00:00:00Z", "older match"), @@ -138,6 +139,8 @@ describe("VictoriaLogs queries", () => { expect(result.hasMore).toBe(true); const url = urls[0]; expect(url?.searchParams.get("limit")).toBe("2"); + expect(url?.searchParams.has("timeout")).toBe(false); + expect(requestSignal).toBeInstanceOf(AbortSignal); const query = url?.searchParams.get("query") || ""; expect(query).toContain("service_id:service-1"); expect(query).toContain("_time:24h"); diff --git a/web/tests/victoria-metrics-service-metrics.test.ts b/web/tests/victoria-metrics-service-metrics.test.ts index ef489e0f..cf44d606 100644 --- a/web/tests/victoria-metrics-service-metrics.test.ts +++ b/web/tests/victoria-metrics-service-metrics.test.ts @@ -11,7 +11,6 @@ const END_TS = Date.parse("2026-07-02T12:30:00Z") / 1000; describe("VictoriaMetrics service metrics", () => { afterEach(() => { - vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.resetModules();