From b0adc29472121b565f70823c19cc5b5c67c8adc6 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Thu, 23 Jul 2026 00:56:35 +1000 Subject: [PATCH 01/43] Add automatic service placement --- cli/internal/cli/app.go | 31 +- cli/internal/cli/app_test.go | 38 ++- cli/internal/manifest/manifest.go | 48 +++ cli/internal/manifest/manifest_test.go | 59 ++++ docs/api/public-api.mdx | 26 +- web/actions/projects.ts | 229 ++++++++++---- web/app/api/projects/[id]/services/route.ts | 12 +- .../service/details/replicas-section.tsx | 245 ++++++++++++--- web/db/schema.ts | 7 + web/lib/agent-status.ts | 3 +- web/lib/agent/expected-state.ts | 23 +- web/lib/deploy-service.ts | 40 ++- web/lib/inngest/functions/crons.ts | 8 + web/lib/inngest/functions/rollout-helpers.ts | 78 ++++- web/lib/inngest/functions/rollout-workflow.ts | 15 +- web/lib/public-api.ts | 221 ++++++++++++-- web/lib/scheduler.ts | 281 +++++++++++++++++- web/lib/service-config.ts | 102 +++++-- web/lib/service-revision-changes.ts | 95 ++++-- web/lib/service-revision-spec.ts | 41 ++- web/lib/service-revisions.ts | 214 ++++++++----- web/tests/autoplacement.test.ts | 54 ++++ web/tests/build-assignment.test.ts | 3 +- web/tests/deploy-service-revision.test.ts | 5 + web/tests/public-api-configuration.test.ts | 31 ++ web/tests/public-api-source.test.ts | 34 +++ web/tests/service-config.test.ts | 6 +- web/tests/service-revision-build.test.ts | 5 +- web/tests/service-revision-changes.test.ts | 20 +- web/tests/service-revision-spec.test.ts | 32 +- web/tests/service-revisions-route.test.ts | 3 +- 31 files changed, 1679 insertions(+), 330 deletions(-) create mode 100644 web/tests/autoplacement.test.ts diff --git a/cli/internal/cli/app.go b/cli/internal/cli/app.go index fd089775..cefd7fc7 100644 --- a/cli/internal/cli/app.go +++ b/cli/internal/cli/app.go @@ -435,7 +435,14 @@ func (a *App) linkCommand() *cobra.Command { IsPublic bool `json:"public"` Domain *string `json:"domain"` } `json:"ports"` - Replicas int `json:"replicas"` + Replicas int `json:"replicas"` + Placement *struct { + Mode string `json:"mode"` + } `json:"placement"` + Placements []struct { + ServerID string `json:"serverId"` + Count int `json:"count"` + } `json:"placements"` HealthCheck *manifest.HealthCheck `json:"healthCheck"` StartCommand *string `json:"startCommand"` Resources *manifest.Resources `json:"resources"` @@ -468,7 +475,19 @@ func (a *App) linkCommand() *cobra.Command { for i, p := range cfg.Current.Ports { ports[i] = manifest.Port{ContainerPort: p.ContainerPort, Public: p.IsPublic, Domain: p.Domain} } - m := manifest.Manifest{APIVersion: "v1", Project: manifest.Project{ID: project.ID, Slug: project.Slug}, Environment: manifest.Environment{ID: environment.ID, Name: environment.Name}, Service: manifest.Service{ID: service.ID, Name: service.Name, Source: service.Source, Hostname: cfg.Current.Hostname, Ports: ports, Replicas: cfg.Current.Replicas, HealthCheck: cfg.Current.HealthCheck, StartCommand: cfg.Current.StartCommand, Resources: cfg.Current.Resources}} + var placement *manifest.Placement + if cfg.Current.Placement != nil { + if cfg.Current.Placement.Mode == "automatic" { + placement = &manifest.Placement{Mode: "automatic"} + } else if len(cfg.Current.Placements) > 0 { + placement = &manifest.Placement{Mode: "manual"} + placement.Servers = make([]manifest.PlacementServer, len(cfg.Current.Placements)) + for i, p := range cfg.Current.Placements { + placement.Servers[i] = manifest.PlacementServer{ServerID: p.ServerID, Count: p.Count} + } + } + } + m := manifest.Manifest{APIVersion: "v1", Project: manifest.Project{ID: project.ID, Slug: project.Slug}, Environment: manifest.Environment{ID: environment.ID, Name: environment.Name}, Service: manifest.Service{ID: service.ID, Name: service.Name, Source: service.Source, Hostname: cfg.Current.Hostname, Ports: ports, Replicas: cfg.Current.Replicas, Placement: placement, HealthCheck: cfg.Current.HealthCheck, StartCommand: cfg.Current.StartCommand, Resources: cfg.Current.Resources}} if err := manifest.Save(manifestPath, m); err != nil { return err } @@ -508,6 +527,14 @@ func (a *App) applyCommand() *cobra.Command { return errors.New("service is not linked: run `tc link`") } body := map[string]any{"source": sourcePatch(loaded.Manifest.Service.Source), "hostname": loaded.Manifest.Service.Hostname, "ports": loaded.Manifest.Service.Ports, "replicas": loaded.Manifest.Service.Replicas, "healthCheck": loaded.Manifest.Service.HealthCheck, "startCommand": loaded.Manifest.Service.StartCommand} + if placement := loaded.Manifest.Service.Placement; placement != nil { + delete(body, "replicas") + if placement.Mode == "automatic" { + body["placement"] = map[string]any{"mode": "automatic", "replicas": loaded.Manifest.Service.Replicas} + } else { + body["placement"] = map[string]any{"mode": "manual", "placements": placement.Servers} + } + } if loaded.Manifest.Service.Resources != nil { body["resources"] = loaded.Manifest.Service.Resources } diff --git a/cli/internal/cli/app_test.go b/cli/internal/cli/app_test.go index c4b691a4..2f135542 100644 --- a/cli/internal/cli/app_test.go +++ b/cli/internal/cli/app_test.go @@ -117,7 +117,7 @@ func TestLinkByIDsFetchesConfigurationAndSupportsPublicGitHub(t *testing.T) { case "/api/v1/projects/p/environments/e/services": w.Write([]byte(`{"services":[{"id":"s","name":"web","source":{"type":"github","repository":"https://github.com/acme/public","branch":"main","rootDir":"cmd/api"}}]}`)) case "/api/v1/projects/p/environments/e/services/s/configuration": - w.Write([]byte(`{"current":{"replicas":2,"hostname":null,"ports":[],"healthCheck":null,"startCommand":null},"management":{"patchable":true,"blockers":[]}}`)) + w.Write([]byte(`{"current":{"replicas":2,"placement":{"mode":"manual"},"placements":[{"serverId":"server-a","count":1},{"serverId":"server-b","count":1}],"hostname":null,"ports":[],"healthCheck":null,"startCommand":null},"management":{"patchable":true,"blockers":[]}}`)) default: t.Errorf("path=%s", r.URL.Path) } @@ -136,6 +136,9 @@ func TestLinkByIDsFetchesConfigurationAndSupportsPublicGitHub(t *testing.T) { if loaded.Manifest.Service.Source.Repository != "https://github.com/acme/public" || loaded.Manifest.Service.Source.RootDir == nil || *loaded.Manifest.Service.Source.RootDir != root || loaded.Manifest.Service.Replicas != 2 { t.Fatalf("manifest=%#v", loaded.Manifest) } + if loaded.Manifest.Service.Placement == nil || loaded.Manifest.Service.Placement.Mode != "manual" || len(loaded.Manifest.Service.Placement.Servers) != 2 || loaded.Manifest.Service.Placement.Servers[1].ServerID != "server-b" { + t.Fatalf("placement=%#v", loaded.Manifest.Service.Placement) + } want := []string{"/api/v1/projects", "/api/v1/projects/p/environments", "/api/v1/projects/p/environments/e/services", "/api/v1/projects/p/environments/e/services/s/configuration"} if !reflect.DeepEqual(paths, want) { t.Fatalf("paths=%v", paths) @@ -182,6 +185,39 @@ func TestApplyExactNestedPatchForSources(t *testing.T) { } } +func TestApplyPlacementPayloads(t *testing.T) { + for _, tc := range []struct { + name, yaml string + want map[string]any + }{ + {"automatic", " placement: {mode: automatic}\n", map[string]any{"mode": "automatic", "replicas": float64(2)}}, + {"manual", " placement:\n mode: manual\n servers:\n - {serverId: server-a, count: 2}\n", map[string]any{"mode": "manual", "placements": []any{map[string]any{"serverId": "server-a", "count": float64(2)}}}}, + } { + t.Run(tc.name, func(t *testing.T) { + configHome(t) + d := t.TempDir() + writeManifest(t, d, imageManifest+tc.yaml) + var body map[string]any + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewDecoder(r.Body).Decode(&body) + w.Write([]byte(`{"action":"updated"}`)) + })) + defer s.Close() + writeConfig(t, s.URL) + app, _ := testApp(t, d, s.Client()) + if err := execute(app, "apply"); err != nil { + t.Fatal(err) + } + if _, exists := body["replicas"]; exists { + t.Fatalf("legacy replicas present: %#v", body) + } + if !reflect.DeepEqual(body["placement"], tc.want) { + t.Fatalf("placement=%#v want=%#v", body["placement"], tc.want) + } + }) + } +} + func TestCollectionRequestsFollowPagination(t *testing.T) { queries := map[string][]string{} s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/cli/internal/manifest/manifest.go b/cli/internal/manifest/manifest.go index 8891aa94..cb1ba4e0 100644 --- a/cli/internal/manifest/manifest.go +++ b/cli/internal/manifest/manifest.go @@ -36,10 +36,19 @@ type Service struct { Hostname *string `json:"hostname" yaml:"hostname"` Ports []Port `json:"ports" yaml:"ports"` Replicas int `json:"replicas" yaml:"replicas"` + Placement *Placement `json:"placement,omitempty" yaml:"placement,omitempty"` HealthCheck *HealthCheck `json:"healthCheck" yaml:"healthCheck"` StartCommand *string `json:"startCommand" yaml:"startCommand"` Resources *Resources `json:"resources,omitempty" yaml:"resources,omitempty"` } +type Placement struct { + Mode string `json:"mode" yaml:"mode"` + Servers []PlacementServer `json:"servers,omitempty" yaml:"servers,omitempty"` +} +type PlacementServer struct { + ServerID string `json:"serverId" yaml:"serverId"` + Count int `json:"count" yaml:"count"` +} type Source struct { Type string `json:"type" yaml:"type"` Image string `json:"image,omitempty" yaml:"image,omitempty"` @@ -140,6 +149,12 @@ func ApplyDefaults(m *Manifest) { if m.Service.Replicas == 0 { m.Service.Replicas = 1 } + if p := m.Service.Placement; p != nil { + p.Mode = strings.ToLower(strings.TrimSpace(p.Mode)) + for i := range p.Servers { + p.Servers[i].ServerID = strings.TrimSpace(p.Servers[i].ServerID) + } + } if h := m.Service.HealthCheck; h != nil { h.Cmd = strings.TrimSpace(h.Cmd) if h.Interval == 0 { @@ -213,6 +228,39 @@ func Validate(m Manifest) error { if m.Service.Replicas < 1 || m.Service.Replicas > 10 { return errors.New("service.replicas must be between 1 and 10") } + if p := m.Service.Placement; p != nil { + switch p.Mode { + case "automatic": + if p.Servers != nil { + return errors.New("service.placement.servers cannot be set for automatic placement") + } + case "manual": + total := 0 + seen := make(map[string]struct{}, len(p.Servers)) + for i, server := range p.Servers { + serverID := strings.TrimSpace(server.ServerID) + if serverID == "" { + return fmt.Errorf("service.placement.servers[%d].serverId cannot be blank", i) + } + if _, exists := seen[serverID]; exists { + return fmt.Errorf("service.placement.servers[%d].serverId must be unique", i) + } + seen[serverID] = struct{}{} + if server.Count < 1 { + return fmt.Errorf("service.placement.servers[%d].count must be positive", i) + } + total += server.Count + } + if total < 1 || total > 10 { + return errors.New("service.placement manual total must be between 1 and 10") + } + if total != m.Service.Replicas { + return errors.New("service.placement manual total must equal service.replicas") + } + default: + return errors.New("service.placement.mode must be automatic or manual") + } + } seenPorts := make(map[int]struct{}, len(m.Service.Ports)) for i, p := range m.Service.Ports { if p.ContainerPort < 1 || p.ContainerPort > 65535 { diff --git a/cli/internal/manifest/manifest_test.go b/cli/internal/manifest/manifest_test.go index 39eaf9a3..5b7318e4 100644 --- a/cli/internal/manifest/manifest_test.go +++ b/cli/internal/manifest/manifest_test.go @@ -20,6 +20,65 @@ func TestDefaultsAndRoundTrip(t *testing.T) { t.Fatalf("got=%#v err=%v", got, e) } } + +func TestPlacementRoundTripAndValidation(t *testing.T) { + m := base() + m.Service.Replicas = 3 + m.Service.Placement = &Placement{Mode: " manual ", Servers: []PlacementServer{{ServerID: " server-a ", Count: 2}, {ServerID: "server-b", Count: 1}}} + b, err := Marshal(m) + if err != nil { + t.Fatal(err) + } + got, err := Parse(b) + if err != nil { + t.Fatal(err) + } + if got.Service.Placement == nil || got.Service.Placement.Mode != "manual" || got.Service.Placement.Servers[0].ServerID != "server-a" { + t.Fatalf("placement=%#v", got.Service.Placement) + } + + tests := []struct { + name string + placement *Placement + replicas int + }{ + {"invalid mode", &Placement{Mode: "random"}, 1}, + {"automatic servers", &Placement{Mode: "automatic", Servers: []PlacementServer{}}, 1}, + {"blank server", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: " ", Count: 1}}}, 1}, + {"duplicate server", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: "a", Count: 1}, {ServerID: "a", Count: 1}}}, 2}, + {"nonpositive count", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: "a", Count: 0}}}, 1}, + {"total exceeds limit", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: "a", Count: 11}}}, 10}, + {"total differs from replicas", &Placement{Mode: "manual", Servers: []PlacementServer{{ServerID: "a", Count: 1}}}, 2}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := base() + m.Service.Replicas = tc.replicas + m.Service.Placement = tc.placement + ApplyDefaults(&m) + if err := Validate(m); err == nil { + t.Fatal("invalid placement accepted") + } + }) + } +} + +func TestPlacementOmittedIsBackwardCompatible(t *testing.T) { + m, err := Parse([]byte(`apiVersion: v1 +project: {slug: app} +environment: {name: prod} +service: + name: web + source: {type: image, image: nginx} + replicas: 2 +`)) + if err != nil { + t.Fatal(err) + } + if m.Service.Placement != nil || m.Service.Replicas != 2 { + t.Fatalf("service=%#v", m.Service) + } +} func TestGitHubCanonical(t *testing.T) { m := base() root := `packages\web` diff --git a/docs/api/public-api.mdx b/docs/api/public-api.mdx index c794565e..136d0a01 100644 --- a/docs/api/public-api.mdx +++ b/docs/api/public-api.mdx @@ -156,7 +156,10 @@ Configuration and revision responses never include secret names, values, or ciph "domain": "cloud.example.com" } ], - "replicas": 1, + "placement": { + "mode": "automatic", + "replicas": 3 + }, "healthCheck": { "cmd": "curl --fail http://localhost:3000/health", "interval": 10, @@ -189,7 +192,26 @@ The API supports these source variants: GitHub repository URLs are canonical HTTPS `github.com` URLs. `rootDir` must stay inside the repository. Use `rootDir: null` to clear an existing build root. Source conversion and GitHub repository switching are not supported. -The API only manages stateless services with HTTP ports. Existing volumes, stateful mode, TCP or UDP ports, TLS passthrough, unsupported placement, or invalid resource limits return a conflict with an actionable code. `replicas` must match the existing server placement. +Use automatic placement for stateless services: + +```json +{ "mode": "automatic", "replicas": 3 } +``` + +Use manual placement to choose exact servers: + +```json +{ + "mode": "manual", + "placements": [ + { "serverId": "server-id", "count": 2 } + ] +} +``` + +Manual placement requires online servers with WireGuard configured. Serverless services require proxy servers. Automatic placement is not available for stateful or volume-backed services. The legacy top-level `replicas` field remains supported when placement is unchanged. It must match the existing manual placement. + +The API only manages stateless services with HTTP ports. Existing volumes, stateful mode, TCP or UDP ports, TLS passthrough, or invalid resource limits return a conflict with an actionable code. ## Deployments and builds diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 2f3ce3da..41e96782 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import cronstrue from "cronstrue"; -import { and, desc, eq, inArray, isNotNull, sql } from "drizzle-orm"; +import { and, desc, eq, inArray, isNotNull, isNull, sql } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { ZodError, z } from "zod"; import { db } from "@/db"; @@ -947,16 +947,20 @@ export async function updateServiceServerlessSettings( .from(serviceReplicas) .innerJoin(servers, eq(serviceReplicas.serverId, servers.id)) .where(eq(serviceReplicas.serviceId, serviceId)); - const totalConfiguredReplicas = configuredReplicas.reduce( - (total, replica) => total + replica.count, - 0, - ); + const totalConfiguredReplicas = + service.placementMode === "automatic" + ? service.replicas + : configuredReplicas.reduce( + (total, replica) => total + replica.count, + 0, + ); if (totalConfiguredReplicas < 1) { throw new Error("Serverless services require at least one replica"); } if ( + service.placementMode === "manual" && configuredReplicas.some( (replica) => replica.count > 0 && !replica.serverIsProxy, ) @@ -985,8 +989,51 @@ export type ServiceConfigUpdate = { healthCheck?: ServiceHealthCheckConfig | null; ports?: { add?: PortConfig[]; remove?: string[] }; replicas?: { serverId: string; count: number }[]; + placement?: + | { mode: "automatic"; replicas: number } + | { + mode: "manual"; + placements: { serverId: string; count: number }[]; + }; }; +const placementInputSchema = z.discriminatedUnion("mode", [ + z.strictObject({ + mode: z.literal("automatic"), + replicas: z.number().int().min(1).max(10), + }), + z + .strictObject({ + mode: z.literal("manual"), + placements: z + .array( + z.strictObject({ + serverId: z.string().min(1), + count: z.number().int().min(1).max(10), + }), + ) + .min(1), + }) + .superRefine((value, context) => { + if ( + new Set(value.placements.map((item) => item.serverId)).size !== + value.placements.length + ) + context.addIssue({ + code: "custom", + message: "Server IDs must be unique", + path: ["placements"], + }); + const total = value.placements.reduce((sum, item) => sum + item.count, 0); + if (total > 10) + context.addIssue({ + code: "custom", + message: "Total replicas must be between 1 and 10", + path: ["placements"], + }); + }), +]); + export async function updateServiceConfig( serviceId: string, config: ServiceConfigUpdate, @@ -1106,33 +1153,73 @@ export async function updateServiceConfig( } } - if (config.replicas) { - const replicas = config.replicas; + if (config.placement || config.replicas) { + const placement = placementInputSchema.parse( + config.placement ?? { + mode: "manual", + placements: config.replicas?.filter((replica) => replica.count > 0), + }, + ); await db.transaction(async (tx) => { await tx.execute( sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, ); const [currentService] = await tx - .select({ serverlessEnabled: services.serverlessEnabled }) + .select({ + serverlessEnabled: services.serverlessEnabled, + stateful: services.stateful, + placementMode: services.placementMode, + }) .from(services) .where(eq(services.id, serviceId)) .limit(1); - if (currentService?.serverlessEnabled) { - const selectedServerIds = replicas - .filter((replica) => replica.count > 0) - .map((replica) => replica.serverId); + if (!currentService) throw new Error("Service not found"); + if (placement.mode === "automatic") { + const volume = await tx + .select({ id: serviceVolumes.id }) + .from(serviceVolumes) + .where(eq(serviceVolumes.serviceId, serviceId)) + .limit(1); + if (currentService.stateful || volume.length > 0) + throw new Error( + "Automatic placement is not supported for stateful services or services with volumes", + ); + await tx + .update(services) + .set({ placementMode: "automatic", replicas: placement.replicas }) + .where(eq(services.id, serviceId)); + await tx + .delete(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + return; + } + + const replicas = placement.placements; + const selectedServerIds = replicas.map((replica) => replica.serverId); + const selectedServers = await tx + .select({ + id: servers.id, + isProxy: servers.isProxy, + status: servers.status, + wireguardIp: servers.wireguardIp, + }) + .from(servers) + .where(inArray(servers.id, selectedServerIds)); + if (selectedServers.length !== selectedServerIds.length) + throw new Error("One or more selected servers do not exist"); + if ( + selectedServers.some( + (server) => server.status !== "online" || !server.wireguardIp, + ) + ) + throw new Error("Manual placement requires online, configured servers"); + if (currentService.serverlessEnabled) { if (selectedServerIds.length > 0) { - const workerServers = await tx - .select({ id: servers.id }) - .from(servers) - .where( - and( - inArray(servers.id, selectedServerIds), - eq(servers.isProxy, false), - ), - ); + const workerServers = selectedServers.filter( + (server) => !server.isProxy, + ); if (workerServers.length > 0) { throw new Error( "Disable serverless before deploying to worker nodes", @@ -1141,6 +1228,13 @@ export async function updateServiceConfig( } } + await tx + .update(services) + .set({ + placementMode: "manual", + replicas: replicas.reduce((sum, replica) => sum + replica.count, 0), + }) + .where(eq(services.id, serviceId)); await tx .delete(serviceReplicas) .where(eq(serviceReplicas.serviceId, serviceId)); @@ -1341,55 +1435,60 @@ export async function addServiceVolume( const validatedName = volumeNameSchema.parse(name); const validatedPath = containerPathSchema.parse(containerPath); - const service = await getService(serviceId); - if (!service) { - throw new Error("Service not found"); - } - - const configuredReplicas = await db - .select({ count: serviceReplicas.count }) - .from(serviceReplicas) - .where(eq(serviceReplicas.serviceId, serviceId)); - const totalReplicas = configuredReplicas.reduce( - (sum, r) => sum + r.count, - 0, - ); - - if (totalReplicas > 1) { - throw new Error( - "Volumes can only be added to services with 1 replica. Reduce replicas to 1 first.", + return await db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, ); - } - - const existing = await db - .select() - .from(serviceVolumes) - .where(eq(serviceVolumes.serviceId, serviceId)); + const service = await tx + .select() + .from(services) + .where(and(eq(services.id, serviceId), isNull(services.deletedAt))) + .then((rows) => rows[0]); + if (!service) throw new Error("Service not found"); + if (service.placementMode === "automatic") { + throw new Error("Switch to manual placement before adding a volume"); + } - if (existing.some((v) => v.name === validatedName)) { - throw new Error("Volume with this name already exists"); - } + const configuredReplicas = await tx + .select({ count: serviceReplicas.count }) + .from(serviceReplicas) + .where(eq(serviceReplicas.serviceId, serviceId)); + const totalReplicas = configuredReplicas.reduce( + (sum, replica) => sum + replica.count, + 0, + ); + if (totalReplicas > 1) { + throw new Error( + "Volumes can only be added to services with 1 replica. Reduce replicas to 1 first.", + ); + } - if (existing.some((v) => v.containerPath === validatedPath)) { - throw new Error("A volume with this container path already exists"); - } + const existing = await tx + .select() + .from(serviceVolumes) + .where(eq(serviceVolumes.serviceId, serviceId)); + if (existing.some((volume) => volume.name === validatedName)) { + throw new Error("Volume with this name already exists"); + } + if (existing.some((volume) => volume.containerPath === validatedPath)) { + throw new Error("A volume with this container path already exists"); + } - const id = randomUUID(); - await db.insert(serviceVolumes).values({ - id, - serviceId, - name: validatedName, - containerPath: validatedPath, + const id = randomUUID(); + await tx.insert(serviceVolumes).values({ + id, + serviceId, + name: validatedName, + containerPath: validatedPath, + }); + if (!service.stateful) { + await tx + .update(services) + .set({ stateful: true }) + .where(eq(services.id, serviceId)); + } + return { id, name: validatedName, containerPath: validatedPath }; }); - - if (!service.stateful) { - await db - .update(services) - .set({ stateful: true }) - .where(eq(services.id, serviceId)); - } - - return { id, name: validatedName, containerPath: validatedPath }; } catch (error) { if (error instanceof ZodError) { throw new Error( diff --git a/web/app/api/projects/[id]/services/route.ts b/web/app/api/projects/[id]/services/route.ts index 5988f756..e6545e0e 100644 --- a/web/app/api/projects/[id]/services/route.ts +++ b/web/app/api/projects/[id]/services/route.ts @@ -25,6 +25,7 @@ import { revisionSpecToDeployedConfig, type SourceConfig, } from "@/lib/service-config"; +import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; export async function GET( request: Request, @@ -154,22 +155,25 @@ export async function GET( .where(eq(serviceRevisions.id, activeDeployment.serviceRevisionId)) .then((rows) => rows[0]) : null; - const revisionServers = activeRevision + const activeSpecification = activeRevision + ? parseServiceRevisionSpec(activeRevision.specification) + : null; + const revisionServers = activeSpecification ? await db .select({ id: servers.id, name: servers.name }) .from(servers) .where( inArray( servers.id, - activeRevision.specification.placements.map( + activeSpecification.placements.map( (placement) => placement.serverId, ), ), ) : []; - const activeConfig = activeRevision + const activeConfig = activeSpecification ? revisionSpecToDeployedConfig( - activeRevision.specification, + activeSpecification, Object.fromEntries( revisionServers.map((server) => [server.id, server.name]), ), diff --git a/web/components/service/details/replicas-section.tsx b/web/components/service/details/replicas-section.tsx index 935f2cde..dfc8491b 100644 --- a/web/components/service/details/replicas-section.tsx +++ b/web/components/service/details/replicas-section.tsx @@ -14,21 +14,27 @@ import { } from "@/components/ui/empty"; import { Input } from "@/components/ui/input"; import { Spinner } from "@/components/ui/spinner"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import type { Server as ServerType, ServiceWithDetails as Service, } from "@/db/types"; -type ServerInfo = Pick; -type ServerWithStatus = ServerInfo & { status: string }; +type ServerInfo = Pick< + ServerType, + "id" | "name" | "isProxy" | "status" | "wireguardIp" +>; +type PlacementMode = "manual" | "automatic"; const fetcher = async (url: string): Promise => { const res = await fetch(url); - const servers: ServerWithStatus[] = await res.json(); - return servers.map(({ id, name, isProxy }) => ({ + const servers: ServerInfo[] = await res.json(); + return servers.map(({ id, name, isProxy, status, wireguardIp }) => ({ id, name, isProxy, + status, + wireguardIp, })); }; @@ -46,6 +52,11 @@ export const ReplicasSection = memo(function ReplicasSection({ {}, ); const [selectedServerId, setSelectedServerId] = useState(null); + const [placementMode, setPlacementMode] = useState( + service.placementMode, + ); + const [desiredReplicas, setDesiredReplicas] = useState(service.replicas); + const [isEditing, setIsEditing] = useState(false); const [isSaving, setIsSaving] = useState(false); const configuredReplicas = useMemo( @@ -64,7 +75,7 @@ export const ReplicasSection = memo(function ReplicasSection({ } else { setSelectedServerId(null); } - } else { + } else if (!isEditing) { const replicaMap: Record = {}; for (const r of configuredReplicas) { replicaMap[r.serverId] = r.count; @@ -75,8 +86,65 @@ export const ReplicasSection = memo(function ReplicasSection({ } } setLocalReplicas(replicaMap); + setPlacementMode(service.placementMode); + setDesiredReplicas(service.replicas); } - }, [servers, configuredReplicas, service.stateful, service.lockedServerId]); + }, [ + servers, + configuredReplicas, + service.stateful, + service.lockedServerId, + service.placementMode, + service.replicas, + isEditing, + ]); + + useEffect(() => { + if (!servers || service.stateful) return; + setLocalReplicas((current) => { + const next = { ...current }; + for (const server of servers) next[server.id] ??= 0; + return next; + }); + }, [servers, service.stateful]); + + useEffect(() => { + if ( + !isEditing || + service.stateful || + placementMode !== service.placementMode + ) + return; + if (placementMode === "automatic" && desiredReplicas === service.replicas) { + setIsEditing(false); + return; + } + if (placementMode === "manual") { + const configuredMap = new Map( + configuredReplicas.map((replica) => [replica.serverId, replica.count]), + ); + const localEntries = Object.entries(localReplicas).filter( + ([, count]) => count > 0, + ); + if ( + localEntries.length === configuredMap.size && + localEntries.every( + ([serverId, count]) => configuredMap.get(serverId) === count, + ) + ) { + setIsEditing(false); + } + } + }, [ + configuredReplicas, + desiredReplicas, + isEditing, + localReplicas, + placementMode, + service.placementMode, + service.replicas, + service.stateful, + ]); const hasChanges = useMemo(() => { if (service.stateful) { @@ -84,6 +152,10 @@ export const ReplicasSection = memo(function ReplicasSection({ configuredReplicas.length > 0 ? configuredReplicas[0].serverId : null; return selectedServerId !== currentServerId; } + if (placementMode !== service.placementMode) return true; + if (placementMode === "automatic") { + return desiredReplicas !== service.replicas; + } const configuredMap = new Map( configuredReplicas.map((r) => [r.serverId, r.count]), @@ -93,7 +165,16 @@ export const ReplicasSection = memo(function ReplicasSection({ if (configured !== count) return true; } return false; - }, [configuredReplicas, localReplicas, service.stateful, selectedServerId]); + }, [ + configuredReplicas, + desiredReplicas, + localReplicas, + placementMode, + service.placementMode, + service.replicas, + service.stateful, + selectedServerId, + ]); const isChangingServer = useMemo(() => { if (!service.stateful || !service.lockedServerId) return false; @@ -112,12 +193,27 @@ export const ReplicasSection = memo(function ReplicasSection({ ]); const updateReplicas = useCallback((serverId: string, value: number) => { + setIsEditing(true); setLocalReplicas((prev) => ({ ...prev, - [serverId]: Math.max(0, Math.min(10, value)), + [serverId]: Math.max(0, Math.min(10, Math.floor(value))), })); }, []); + const handleModeChange = (mode: string) => { + const nextMode = mode as PlacementMode; + if (nextMode === placementMode) return; + setIsEditing(true); + if (nextMode === "automatic") { + const manualTotal = Object.values(localReplicas).reduce( + (sum, count) => sum + count, + 0, + ); + setDesiredReplicas(Math.max(1, Math.min(10, manualTotal || 1))); + } + setPlacementMode(nextMode); + }; + const handleSave = async () => { setIsSaving(true); try { @@ -126,12 +222,25 @@ export const ReplicasSection = memo(function ReplicasSection({ replicas = selectedServerId ? [{ serverId: selectedServerId, count: 1 }] : []; - } else { + } else if (placementMode === "manual") { replicas = Object.entries(localReplicas) .filter(([, count]) => count > 0) .map(([serverId, count]) => ({ serverId, count })); + await updateServiceConfig(service.id, { + placement: { mode: "manual", placements: replicas }, + }); + onUpdate(); + return; + } else { + await updateServiceConfig(service.id, { + placement: { mode: "automatic", replicas: desiredReplicas }, + }); + onUpdate(); + return; } - await updateServiceConfig(service.id, { replicas }); + await updateServiceConfig(service.id, { + placement: { mode: "manual", placements: replicas }, + }); onUpdate(); } finally { setIsSaving(false); @@ -148,11 +257,16 @@ export const ReplicasSection = memo(function ReplicasSection({ const workerUnavailableReason = service.serverlessEnabled ? "Disable serverless before deploying to worker nodes" : null; + const manualServerUnavailableReason = (server: ServerInfo) => { + if (server.status !== "online" || !server.wireguardIp) { + return "Server must be online and configured before placement"; + } + return service.serverlessEnabled && !server.isProxy + ? workerUnavailableReason + : null; + }; - const configuredTotal = configuredReplicas.reduce( - (sum, r) => sum + r.count, - 0, - ); + const manualTotalIsValid = totalReplicas >= 1 && totalReplicas <= 10; if (service.stateful) { return ( @@ -215,12 +329,8 @@ export const ReplicasSection = memo(function ReplicasSection({ type="button" key={server.id} onClick={() => setSelectedServerId(server.id)} - disabled={service.serverlessEnabled && !server.isProxy} - title={ - service.serverlessEnabled && !server.isProxy - ? workerUnavailableReason || undefined - : undefined - } + disabled={!!manualServerUnavailableReason(server)} + title={manualServerUnavailableReason(server) || undefined} className={`flex items-center gap-4 p-3 rounded-md text-left transition-colors ${ selectedServerId === server.id ? "bg-primary text-primary-foreground" @@ -264,15 +374,67 @@ export const ReplicasSection = memo(function ReplicasSection({ return ( 0 - ? `${configuredTotal} replica${configuredTotal !== 1 ? "s" : ""}` - : "No replicas" - } - summaryMuted={configuredTotal === 0} + summary={`${service.placementMode === "automatic" ? "Automatic" : "Manual"} · ${service.replicas} desired`} + summaryMuted={service.replicas === 0} >
- {isLoading ? ( + + + + Automatic + + + Manual + + + + + {placementMode === "automatic" ? ( +
+
+ +

+ The control plane distributes replicas evenly across healthy + {service.serverlessEnabled ? " proxy nodes" : " nodes"} and + moves them after failures. +

+
+ { + setIsEditing(true); + setDesiredReplicas( + Math.max( + 1, + Math.min(10, Math.floor(event.target.valueAsNumber || 1)), + ), + ); + }} + className="w-24" + aria-describedby="automatic-replica-range" + /> +

+ Choose between 1 and 10 replicas. +

+ {hasChanges ? ( +
+ +
+ ) : null} +
+ ) : isLoading ? (
@@ -283,7 +445,8 @@ export const ReplicasSection = memo(function ReplicasSection({ No online servers available - Add a server to deploy this service. + Add a server to place replicas manually, or use automatic + placement. ) : ( @@ -292,11 +455,7 @@ export const ReplicasSection = memo(function ReplicasSection({ {servers.map((server) => (
@@ -316,14 +475,12 @@ export const ReplicasSection = memo(function ReplicasSection({ (localReplicas[server.id] || 0) - 1, ) } - disabled={ - (localReplicas[server.id] || 0) <= 0 || - (service.serverlessEnabled && !server.isProxy) - } + disabled={(localReplicas[server.id] || 0) <= 0} > - @@ -332,7 +489,7 @@ export const ReplicasSection = memo(function ReplicasSection({ parseInt(e.target.value, 10) || 0, ) } - disabled={service.serverlessEnabled && !server.isProxy} + disabled={!!manualServerUnavailableReason(server)} min={0} max={10} className="w-16 h-8 text-center" @@ -349,7 +506,7 @@ export const ReplicasSection = memo(function ReplicasSection({ } disabled={ (localReplicas[server.id] || 0) >= 10 || - (service.serverlessEnabled && !server.isProxy) + !!manualServerUnavailableReason(server) } > + @@ -364,14 +521,18 @@ export const ReplicasSection = memo(function ReplicasSection({ {totalReplicas !== 1 ? "s" : ""}
- {totalReplicas === 0 && ( + {!manualTotalIsValid && (

- Add at least 1 replica to deploy + Manual placement requires 1 to 10 replicas in total.

)} {hasChanges && (
-
diff --git a/web/db/schema.ts b/web/db/schema.ts index 96646da1..bac1f16f 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -330,6 +330,7 @@ export const servers = pgTable("servers", { .notNull() .default("pending"), lastHeartbeat: timestamp("last_heartbeat", { withTimezone: true }), + onlineSince: timestamp("online_since", { withTimezone: true }), resourcesCpu: integer("resources_cpu"), resourcesMemory: integer("resources_memory"), resourcesDisk: integer("resources_disk"), @@ -404,6 +405,12 @@ export const services = pgTable( githubBranch: text("github_branch").default("main"), githubRootDir: text("github_root_dir"), replicas: integer("replicas").notNull().default(1), + placementMode: text("placement_mode", { enum: ["manual", "automatic"] }) + .notNull() + .default("manual"), + lastAutomaticPlacementAt: timestamp("last_automatic_placement_at", { + withTimezone: true, + }), stateful: boolean("stateful").notNull().default(false), lockedServerId: text("locked_server_id").references(() => servers.id, { onDelete: "set null", diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index d4f3f291..abd7940b 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, isNotNull, isNull, or } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; import { db } from "@/db"; import { type AgentHealth, @@ -651,6 +651,7 @@ export async function applyStatusReport( const updateData: Record = { lastHeartbeat: new Date(), status: "online", + onlineSince: sql`case when ${servers.status} = 'online' and ${servers.onlineSince} is not null then ${servers.onlineSince} else now() end`, }; let completedAgentUpgradeTarget: string | null = null; diff --git a/web/lib/agent/expected-state.ts b/web/lib/agent/expected-state.ts index 0e4a0b5b..10bf8c77 100644 --- a/web/lib/agent/expected-state.ts +++ b/web/lib/agent/expected-state.ts @@ -16,10 +16,10 @@ import { runtimeExpectedStates, } from "@/lib/deployment-status"; import { selectRoutingSyncRolloutIds } from "@/lib/routing-sync"; -import { - SERVICE_REVISION_SCHEMA_VERSION, - type ServiceRevisionSecret, - type ServiceRevisionSpec, +import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import type { + ServiceRevisionSecret, + ServiceRevisionSpec, } from "@/lib/service-revision-spec"; import { getWireGuardPeers } from "@/lib/wireguard"; @@ -241,9 +241,7 @@ async function getRuntimeServiceRevisions(): Promise { const runtimeServices = new Map(); for (const row of rows) { - if (row.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION) { - throw new Error(`Service ${row.serviceId} uses an unsupported revision`); - } + const specification = parseServiceRevisionSpec(row.specification); const existing = runtimeServices.get(row.serviceId); if (existing && existing.revisionId !== row.revisionId) { throw new Error(`Service ${row.serviceId} has multiple active revisions`); @@ -252,7 +250,7 @@ async function getRuntimeServiceRevisions(): Promise { id: row.serviceId, name: row.serviceName, revisionId: row.revisionId, - specification: row.specification, + specification, }); } return [...runtimeServices.values()].sort((a, b) => a.id.localeCompare(b.id)); @@ -352,14 +350,7 @@ export function buildExpectedContainersFromRows({ if (!revision) { throw new Error(`Deployment ${dep.id} has no service revision`); } - if ( - revision.specification.schemaVersion !== SERVICE_REVISION_SCHEMA_VERSION - ) { - throw new Error( - `Deployment ${dep.id} uses an unsupported service revision`, - ); - } - const specification = revision.specification; + const specification = parseServiceRevisionSpec(revision.specification); const ports = (portsByDeploymentId.get(dep.id) ?? []) .slice() .sort( diff --git a/web/lib/deploy-service.ts b/web/lib/deploy-service.ts index e7825137..55d05bbf 100644 --- a/web/lib/deploy-service.ts +++ b/web/lib/deploy-service.ts @@ -1,4 +1,4 @@ -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { db } from "@/db"; import { getService } from "@/db/queries"; @@ -25,17 +25,31 @@ export async function deployServiceRevisionInternal( ); if (!result.rolloutId) return result; - await inngest.send( - inngestEvents.rolloutCreated.create( - { - rolloutId: result.rolloutId, - serviceId, - }, - { - id: `rollout-created-${result.rolloutId}`, - }, - ), - ); + try { + await inngest.send( + inngestEvents.rolloutCreated.create( + { + rolloutId: result.rolloutId, + serviceId, + }, + { + id: `rollout-created-${result.rolloutId}`, + }, + ), + ); + } catch (error) { + await db + .update(rollouts) + .set({ + status: "failed", + currentStage: "enqueue_failed", + completedAt: new Date(), + }) + .where( + and(eq(rollouts.id, result.rolloutId), eq(rollouts.status, "queued")), + ); + throw error; + } return result; } @@ -116,7 +130,7 @@ export async function deployServiceInternal( currentStage: "enqueue_failed", completedAt: new Date(), }) - .where(eq(rollouts.id, rolloutId)); + .where(and(eq(rollouts.id, rolloutId), eq(rollouts.status, "queued"))); throw error; } diff --git a/web/lib/inngest/functions/crons.ts b/web/lib/inngest/functions/crons.ts index 92c3a068..f64e12e2 100644 --- a/web/lib/inngest/functions/crons.ts +++ b/web/lib/inngest/functions/crons.ts @@ -10,6 +10,8 @@ import { checkAndRunScheduledDeployments, cleanupStaleItems, failTimedOutAgentUpgrades, + rebalanceAutomaticServices, + recoverInvalidAutomaticPlacements, } from "@/lib/scheduler"; import { inngest } from "../client"; @@ -24,6 +26,12 @@ export const staleServerCheck = inngest.createFunction( console.log("[cron] running stale server check"); await checkAndRecoverStaleServers(); }); + await step.run("recover-invalid-automatic-placements", async () => { + await recoverInvalidAutomaticPlacements(); + }); + await step.run("rebalance-automatic-services", async () => { + await rebalanceAutomaticServices(); + }); }, ); diff --git a/web/lib/inngest/functions/rollout-helpers.ts b/web/lib/inngest/functions/rollout-helpers.ts index 3943b9f6..46bb7b44 100644 --- a/web/lib/inngest/functions/rollout-helpers.ts +++ b/web/lib/inngest/functions/rollout-helpers.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, lt } from "drizzle-orm"; import { db } from "@/db"; import { deploymentPorts, @@ -15,9 +15,28 @@ import { enqueueWork } from "@/lib/work-queue"; const PORT_RANGE_START = 30000; const PORT_RANGE_END = 32767; +export const AUTOMATIC_PLACEMENT_SERVER_STABILIZATION_MS = 10 * 60 * 1000; export type Placement = { serverId: string; replicas: number }; +export function distributeReplicas( + serverIds: string[], + replicas: number, +): Placement[] { + const ids = [...new Set(serverIds)].sort((a, b) => a.localeCompare(b)); + if (ids.length === 0) throw new Error("No eligible servers for deployment"); + if (!Number.isInteger(replicas) || replicas < 1 || replicas > 10) + throw new Error("Replica count must be between 1 and 10"); + const counts = new Map(ids.map((id) => [id, 0])); + for (let index = 0; index < replicas; index++) { + const id = ids[index % ids.length]; + counts.set(id, (counts.get(id) ?? 0) + 1); + } + return ids + .map((serverId) => ({ serverId, replicas: counts.get(serverId) ?? 0 })) + .filter((placement) => placement.replicas > 0); +} + export type DeploymentContext = { revisionId: string; specification: ServiceRevisionSpec; @@ -109,6 +128,57 @@ export function calculateRevisionPlacements( return { placements, totalReplicas }; } +export async function resolveRevisionPlacements( + specification: ServiceRevisionSpec, +): Promise<{ placements: Placement[]; totalReplicas: number }> { + if (specification.placement.mode === "manual") { + const result = calculateRevisionPlacements(specification); + if (specification.serverless.enabled) { + const selected = await db + .select({ id: servers.id, isProxy: servers.isProxy }) + .from(servers) + .where( + inArray( + servers.id, + result.placements.map((p) => p.serverId), + ), + ); + if ( + selected.length !== result.placements.length || + selected.some((server) => !server.isProxy) + ) + throw new Error( + "Serverless services can only be placed on proxy servers", + ); + } + return result; + } + const eligible = await db + .select({ id: servers.id }) + .from(servers) + .where( + and( + eq(servers.status, "online"), + isNotNull(servers.wireguardIp), + isNotNull(servers.onlineSince), + lt( + servers.onlineSince, + new Date(Date.now() - AUTOMATIC_PLACEMENT_SERVER_STABILIZATION_MS), + ), + ...(specification.serverless.enabled + ? [eq(servers.isProxy, true)] + : []), + ), + ); + return { + placements: distributeReplicas( + eligible.map((server) => server.id), + specification.placement.replicas, + ), + totalReplicas: specification.placement.replicas, + }; +} + export async function validateServers( placements: Placement[], ): Promise< @@ -295,7 +365,7 @@ export async function completeRollout( serviceId: string, context: Omit, ): Promise<{ completed: boolean; stoppedCount: number }> { - const { placements, specification, totalReplicas, isRollingUpdate } = context; + const { placements, specification, isRollingUpdate } = context; const lockedServerId = specification.stateful ? placements[0]?.serverId : undefined; @@ -330,7 +400,9 @@ export async function completeRollout( await tx .update(services) .set({ - replicas: totalReplicas, + ...(specification.placement.mode === "automatic" + ? { lastAutomaticPlacementAt: new Date() } + : {}), ...(lockedServerId ? { lockedServerId } : {}), }) .where(eq(services.id, serviceId)); diff --git a/web/lib/inngest/functions/rollout-workflow.ts b/web/lib/inngest/functions/rollout-workflow.ts index 96c8a1d7..c31c89f8 100644 --- a/web/lib/inngest/functions/rollout-workflow.ts +++ b/web/lib/inngest/functions/rollout-workflow.ts @@ -9,13 +9,13 @@ import { ingestRolloutLog } from "@/lib/victoria-logs"; import { inngest } from "../client"; import { inngestEvents } from "../events"; import { - calculateRevisionPlacements, checkForRollingUpdate, cleanupExistingDeployments, cleanupTerminalDeployments, completeRollout, createDeploymentRecords, issueCertificatesForRevision, + resolveRevisionPlacements, validateServers, } from "./rollout-helpers"; import { handleRolloutFailure } from "./rollout-utils"; @@ -62,7 +62,11 @@ async function acquireRolloutTurn( await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`); const rollout = await tx - .select({ status: rollouts.status, createdAt: rollouts.createdAt }) + .select({ + status: rollouts.status, + currentStage: rollouts.currentStage, + createdAt: rollouts.createdAt, + }) .from(rollouts) .where(eq(rollouts.id, rolloutId)) .then((rows) => rows[0]); @@ -71,7 +75,9 @@ async function acquireRolloutTurn( throw new Error("Rollout not found"); } - if (rollout.status !== "queued") { + const recoverableEnqueueFailure = + rollout.status === "failed" && rollout.currentStage === "enqueue_failed"; + if (rollout.status !== "queued" && !recoverableEnqueueFailure) { return rollout.status === "in_progress" ? "acquired" : "terminal"; } @@ -104,6 +110,7 @@ async function acquireRolloutTurn( .set({ status: "in_progress", currentStage: "preparing", + completedAt: null, }) .where(eq(rollouts.id, rolloutId)); @@ -219,7 +226,7 @@ export const rolloutWorkflow = inngest.createFunction( const placementResult = await step.run("load-placements", async () => { try { - const result = calculateRevisionPlacements(specification); + const result = await resolveRevisionPlacements(specification); await ingestRolloutLog( rolloutId, serviceId, diff --git a/web/lib/public-api.ts b/web/lib/public-api.ts index c782adf7..0a47004b 100644 --- a/web/lib/public-api.ts +++ b/web/lib/public-api.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { and, desc, eq, isNull, ne } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, ne, sql } from "drizzle-orm"; import { z } from "zod"; import { db } from "@/db"; import { @@ -219,7 +219,6 @@ function getManagementBlockers(input: { source: PublicSource; ports: Array; volumeCount: number; - replicaCount: number; }): ManagementBlocker[] { const blockers: ManagementBlocker[] = []; if (input.service.stateful || input.volumeCount > 0) { @@ -252,12 +251,6 @@ function getManagementBlockers(input: { "Public HTTP ports without domains must be managed in the web UI", }); } - if (input.replicaCount < 1 || input.replicaCount > 10) { - blockers.push({ - code: "INVALID_PLACEMENT", - message: "Manual placement must contain between 1 and 10 replicas", - }); - } if ( (input.service.resourceCpuLimit === null) !== (input.service.resourceMemoryLimitMb === null) @@ -286,6 +279,14 @@ function getManagementBlockers(input: { function sanitizeSpec(specification: unknown) { const spec = parseServiceRevisionSpec(specification); + const replicas = + spec.placement.mode === "automatic" + ? spec.placement.replicas + : spec.placements.reduce((sum, placement) => sum + placement.count, 0); + const placement = + spec.placement.mode === "automatic" + ? { mode: "automatic" as const, replicas } + : { mode: "manual" as const, placements: spec.placements, replicas }; return { source: spec.source.type === "github" @@ -293,17 +294,13 @@ function sanitizeSpec(specification: unknown) { type: "github" as const, repository: spec.source.repository, branch: spec.source.branch, - ...(spec.source.rootDir - ? { rootDir: spec.source.rootDir } - : {}), + ...(spec.source.rootDir ? { rootDir: spec.source.rootDir } : {}), } : { type: "image" as const, image: spec.source.image }, hostname: spec.hostname, stateful: spec.stateful, - replicas: spec.placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ), + placement, + replicas, placements: spec.placements, healthCheck: spec.healthCheck, startCommand: spec.startCommand, @@ -382,16 +379,25 @@ export async function safeConfiguration(service: NestedService) { a.name.localeCompare(b.name, "en") || a.containerPath.localeCompare(b.containerPath, "en"), ); - const replicaCount = sortedPlacements.reduce( - (sum, placement) => sum + placement.count, - 0, - ); + const replicaCount = + service.placementMode === "automatic" + ? service.replicas + : sortedPlacements.reduce((sum, placement) => sum + placement.count, 0); + const placement = + service.placementMode === "automatic" + ? { mode: "automatic" as const, replicas: replicaCount } + : { + mode: "manual" as const, + placements: sortedPlacements, + replicas: replicaCount, + }; const current = { source, hostname: service.hostname, stateful: service.stateful, replicas: replicaCount, placements: sortedPlacements, + placement, healthCheck: service.healthCheckCmd ? { cmd: service.healthCheckCmd, @@ -454,11 +460,15 @@ export async function safeConfiguration(service: NestedService) { hostname: current.hostname?.trim() || getDefaultServiceHostname(service.name), stateful: current.stateful, - replicas: current.replicas, - placements: current.placements.map(({ serverId, count }) => ({ - serverId, - count, - })), + placement: + current.placement.mode === "automatic" + ? current.placement + : { + ...current.placement, + placements: current.placement.placements.map( + ({ serverId, count }) => ({ serverId, count }), + ), + }, healthCheck: current.healthCheck, startCommand: current.startCommand?.trim() || null, resources: current.resources, @@ -486,7 +496,6 @@ export async function safeConfiguration(service: NestedService) { source, ports, volumeCount: volumes.length, - replicaCount, }); return { @@ -546,11 +555,47 @@ const hostnameSchema = z /^[a-z0-9]+(?:-[a-z0-9]+)*$/, "hostname must contain only lowercase letters, numbers, and hyphens", ); +export const placementSchema = z.discriminatedUnion("mode", [ + z.strictObject({ + mode: z.literal("automatic"), + replicas: z.number().int().min(1).max(10), + }), + z + .strictObject({ + mode: z.literal("manual"), + placements: z + .array( + z.strictObject({ + serverId: z.string().min(1), + count: z.number().int().min(1).max(10), + }), + ) + .min(1), + }) + .superRefine((value, context) => { + if ( + new Set(value.placements.map((item) => item.serverId)).size !== + value.placements.length + ) + context.addIssue({ + code: "custom", + message: "Server IDs must be unique", + path: ["placements"], + }); + if (value.placements.reduce((sum, item) => sum + item.count, 0) > 10) + context.addIssue({ + code: "custom", + message: "Total replicas must be between 1 and 10", + path: ["placements"], + }); + }), +]); export const configurationPatchSchema = z.strictObject({ source: publicSourceSchema.optional(), hostname: hostnameSchema.nullable().optional(), ports: z.array(portSchema).max(100).optional(), replicas: z.number().int().min(1).max(10).optional(), + placement: placementSchema.optional(), healthCheck: healthCheckSchema.nullable().optional(), startCommand: z.string().trim().min(1).max(4096).nullable().optional(), resources: z @@ -610,6 +655,9 @@ export async function patchConfiguration( } return db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtext(${service.id}))`, + ); const persisted = await tx .select() .from(services) @@ -638,17 +686,26 @@ export async function patchConfiguration( .limit(1) .then((rows) => rows[0]), ]); - const replicaCount = placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ); + const replicaCount = + persisted.placementMode === "automatic" + ? persisted.replicas + : placements.reduce((sum, placement) => sum + placement.count, 0); const source = resolvePersistedSourceFromRows(persisted, repo); + if ( + input.placement?.mode === "automatic" && + (persisted.stateful || volumes.length > 0) + ) { + domainError( + "Automatic placement is not supported for stateful services or services with volumes", + "AUTOMATIC_PLACEMENT_UNSUPPORTED", + 400, + ); + } const blockers = getManagementBlockers({ service: persisted, source, ports, volumeCount: volumes.length, - replicaCount, }); if (blockers[0]) { domainError(blockers[0].message, blockers[0].code); @@ -677,12 +734,71 @@ export async function patchConfiguration( ); } } - if (input.replicas !== undefined && input.replicas !== replicaCount) { + if ( + input.replicas !== undefined && + input.placement && + input.replicas !== + (input.placement.mode === "automatic" + ? input.placement.replicas + : input.placement.placements.reduce( + (sum, item) => sum + item.count, + 0, + )) + ) { + domainError( + "replicas and placement describe different desired replica counts", + "REPLICA_PLACEMENT_MISMATCH", + 400, + ); + } + if ( + input.replicas !== undefined && + !input.placement && + persisted.placementMode === "manual" && + input.replicas !== replicaCount + ) { domainError( `replicas must match the current manual placement of ${replicaCount}`, "REPLICA_PLACEMENT_MISMATCH", ); } + if (input.placement?.mode === "manual") { + const ids = input.placement.placements.map((item) => item.serverId); + const selected = await tx + .select({ + id: servers.id, + isProxy: servers.isProxy, + status: servers.status, + wireguardIp: servers.wireguardIp, + }) + .from(servers) + .where(inArray(servers.id, ids)); + if (selected.length !== ids.length) + domainError( + "One or more selected servers do not exist", + "INVALID_PLACEMENT", + 400, + ); + if ( + selected.some( + (server) => server.status !== "online" || !server.wireguardIp, + ) + ) + domainError( + "Manual placement requires online, configured servers", + "INVALID_PLACEMENT", + 400, + ); + if ( + persisted.serverlessEnabled && + selected.some((server) => !server.isProxy) + ) + domainError( + "Serverless services can only be deployed to proxy nodes", + "SERVERLESS_PROXY_REQUIRED", + 400, + ); + } if (input.hostname) { const duplicate = await tx @@ -808,6 +924,49 @@ export async function patchConfiguration( ) { set.image = input.source.image; } + if (input.placement) { + const desiredReplicas = + input.placement.mode === "automatic" + ? input.placement.replicas + : input.placement.placements.reduce( + (sum, item) => sum + item.count, + 0, + ); + if ( + changed( + "placement", + persisted.placementMode === "automatic" + ? { mode: "automatic", replicas: persisted.replicas } + : { + mode: "manual", + placements: placements + .map(({ serverId, count }) => ({ serverId, count })) + .toSorted((a, b) => a.serverId.localeCompare(b.serverId)), + }, + input.placement, + ) + ) { + set.placementMode = input.placement.mode; + set.replicas = desiredReplicas; + await tx + .delete(serviceReplicas) + .where(eq(serviceReplicas.serviceId, service.id)); + if (input.placement.mode === "manual") + await tx.insert(serviceReplicas).values( + input.placement.placements.map((item) => ({ + id: randomUUID(), + serviceId: service.id, + ...item, + })), + ); + } + } else if ( + input.replicas !== undefined && + persisted.placementMode === "automatic" && + changed("placement.replicas", persisted.replicas, input.replicas) + ) { + set.replicas = input.replicas; + } if (input.source?.type === "github") { const effectiveBranch = repo?.deployBranch || diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index d7c31ea0..1e9e0f66 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -5,6 +5,7 @@ import { deployments, rollouts, servers, + serviceRevisions, services, workQueue, } from "@/db/schema"; @@ -20,12 +21,250 @@ import { sendManualRecoveryRequiredAlert, sendServerOfflineAlert, } from "@/lib/email"; +import { inngest } from "@/lib/inngest/client"; +import { inngestEvents } from "@/lib/inngest/events"; +import { + AUTOMATIC_PLACEMENT_SERVER_STABILIZATION_MS, + distributeReplicas, + resolveRevisionPlacements, +} from "@/lib/inngest/functions/rollout-helpers"; +import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; +import { cloneActiveRevisionAndQueueSystemRollout } from "@/lib/service-revisions"; import { WORK_QUEUE_LEASE_DURATION_MS, WORK_QUEUE_MAX_ATTEMPTS, } from "@/lib/work-queue"; const STALE_THRESHOLD_MS = 75 * SECOND_IN_MILLISECONDS; +export const AUTOMATIC_PLACEMENT_COOLDOWN_MS = 30 * MINUTE_IN_MILLISECONDS; +export const MAX_REBALANCES_PER_RUN = 5; + +async function enqueueSystemRollout( + serviceId: string, + result: { rolloutId: string; created: boolean }, +): Promise { + if (!result.created) return; + try { + await inngest.send( + inngestEvents.rolloutCreated.create( + { rolloutId: result.rolloutId, serviceId }, + { id: `rollout-created-${result.rolloutId}` }, + ), + ); + } catch (error) { + await db + .update(rollouts) + .set({ + status: "failed", + currentStage: "enqueue_failed", + completedAt: new Date(), + }) + .where( + and(eq(rollouts.id, result.rolloutId), eq(rollouts.status, "queued")), + ); + throw error; + } +} + +export async function rebalanceAutomaticServices(): Promise { + const now = new Date(); + const candidates = await db + .select() + .from(services) + .where(and(isNull(services.deletedAt))) + .orderBy(services.id); + let queuedCount = 0; + for (const service of candidates) { + if (queuedCount >= MAX_REBALANCES_PER_RUN) break; + if ( + service.lastAutomaticPlacementAt && + now.getTime() - service.lastAutomaticPlacementAt.getTime() < + AUTOMATIC_PLACEMENT_COOLDOWN_MS + ) + continue; + const [pending, activeDeployments] = await Promise.all([ + db + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, service.id), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .limit(1) + .then((r) => r[0]), + db + .select({ + revisionId: deployments.serviceRevisionId, + specification: serviceRevisions.specification, + }) + .from(deployments) + .innerJoin( + serviceRevisions, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) + .where( + and( + eq(deployments.serviceId, service.id), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + ), + ) + .then((rows) => rows), + ]); + const activeRevisionIds = new Set( + activeDeployments.map((deployment) => deployment.revisionId), + ); + if (pending || activeRevisionIds.size !== 1) continue; + const active = activeDeployments[0]; + if (!active) continue; + const spec = parseServiceRevisionSpec(active.specification); + if (spec.stateful || spec.placement.mode !== "automatic") continue; + const eligible = await db + .select({ id: servers.id }) + .from(servers) + .where( + and( + eq(servers.status, "online"), + isNotNull(servers.wireguardIp), + isNotNull(servers.onlineSince), + lt( + servers.onlineSince, + subtractMilliseconds( + now, + AUTOMATIC_PLACEMENT_SERVER_STABILIZATION_MS, + ), + ), + ...(spec.serverless.enabled ? [eq(servers.isProxy, true)] : []), + ), + ); + if (!eligible.length) continue; + const ideal = distributeReplicas( + eligible.map((s) => s.id), + spec.placement.replicas, + ); + const desired = await db + .select({ + serverId: deployments.serverId, + count: sql`count(*)::int`, + }) + .from(deployments) + .where( + and( + eq(deployments.serviceId, service.id), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + ), + ) + .groupBy(deployments.serverId); + const normalize = ( + rows: Array<{ serverId: string; count?: number; replicas?: number }>, + ) => rows.map((r) => [r.serverId, r.count ?? r.replicas]).sort(); + if (JSON.stringify(normalize(ideal)) === JSON.stringify(normalize(desired))) + continue; + try { + const result = await cloneActiveRevisionAndQueueSystemRollout( + service.id, + active.revisionId, + ); + if (!result.created) continue; + await enqueueSystemRollout(service.id, result); + queuedCount++; + } catch (error) { + console.error(`[scheduler] failed to rebalance ${service.name}`, error); + } + } +} + +export async function recoverInvalidAutomaticPlacements(): Promise { + const activeDeployments = await db + .select({ + serviceId: deployments.serviceId, + serviceName: services.name, + revisionId: deployments.serviceRevisionId, + specification: serviceRevisions.specification, + serverStatus: servers.status, + serverWireguardIp: servers.wireguardIp, + serverIsProxy: servers.isProxy, + serverOnlineSince: servers.onlineSince, + }) + .from(deployments) + .innerJoin(services, eq(services.id, deployments.serviceId)) + .innerJoin(servers, eq(servers.id, deployments.serverId)) + .innerJoin( + serviceRevisions, + eq(serviceRevisions.id, deployments.serviceRevisionId), + ) + .where( + and( + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + isNull(services.deletedAt), + ), + ); + + const byService = new Map(); + for (const deployment of activeDeployments) { + const current = byService.get(deployment.serviceId) ?? []; + current.push(deployment); + byService.set(deployment.serviceId, current); + } + + for (const [serviceId, serviceDeployments] of byService) { + const revisionIds = new Set( + serviceDeployments.map((deployment) => deployment.revisionId), + ); + if (revisionIds.size !== 1) { + console.error( + `[scheduler] skipping automatic recovery for ${serviceId}: multiple active revisions`, + ); + continue; + } + const active = serviceDeployments[0]; + if (!active) continue; + const specification = parseServiceRevisionSpec(active.specification); + if (specification.stateful || specification.placement.mode !== "automatic") + continue; + const hasInvalidPlacement = serviceDeployments.some( + (deployment) => + deployment.serverStatus !== "online" || + !deployment.serverWireguardIp || + !deployment.serverOnlineSince || + deployment.serverOnlineSince.getTime() >= + Date.now() - AUTOMATIC_PLACEMENT_SERVER_STABILIZATION_MS || + (specification.serverless.enabled && !deployment.serverIsProxy), + ); + if (!hasInvalidPlacement) continue; + + const pending = await db + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .limit(1) + .then((rows) => rows[0]); + if (pending) continue; + + try { + await resolveRevisionPlacements(specification); + const result = await cloneActiveRevisionAndQueueSystemRollout( + serviceId, + active.revisionId, + ); + await enqueueSystemRollout(serviceId, result); + } catch (error) { + console.error( + `[scheduler] failed level-triggered recovery for ${active.serviceName}`, + error, + ); + } + } +} async function triggerRecoveryForOfflineServers( offlineServerIds: string[], @@ -40,15 +279,22 @@ async function triggerRecoveryForOfflineServers( serverPublicIp: servers.publicIp, serverWireguardIp: servers.wireguardIp, serviceName: services.name, + serviceId: services.id, + serviceRevisionId: deployments.serviceRevisionId, + specification: serviceRevisions.specification, }) .from(deployments) .innerJoin(servers, eq(servers.id, deployments.serverId)) .innerJoin(services, eq(services.id, deployments.serviceId)) + .innerJoin( + serviceRevisions, + eq(serviceRevisions.id, deployments.serviceRevisionId), + ) .where( and( inArray(deployments.serverId, offlineServerIds), inArray(deployments.runtimeDesiredState, ["running", "stopped"]), - inArray(deployments.trafficState, ["candidate", "active"]), + eq(deployments.trafficState, "active"), isNull(services.deletedAt), ), ); @@ -59,6 +305,32 @@ async function triggerRecoveryForOfflineServers( ); return; } + const automaticallyRecovered = new Set(); + const recoveryAttempted = new Set(); + for (const deployment of affectedDeployments) { + if (recoveryAttempted.has(deployment.serviceId)) continue; + recoveryAttempted.add(deployment.serviceId); + try { + const specification = parseServiceRevisionSpec(deployment.specification); + if ( + specification.stateful || + specification.placement.mode !== "automatic" + ) + continue; + await resolveRevisionPlacements(specification); + const queued = await cloneActiveRevisionAndQueueSystemRollout( + deployment.serviceId, + deployment.serviceRevisionId, + ); + await enqueueSystemRollout(deployment.serviceId, queued); + automaticallyRecovered.add(deployment.serviceId); + } catch (error) { + console.error( + `[scheduler] automatic recovery failed for ${deployment.serviceName}; falling back to manual alert`, + error, + ); + } + } const affectedByServer = new Map< string, @@ -71,6 +343,7 @@ async function triggerRecoveryForOfflineServers( >(); for (const deployment of affectedDeployments) { + if (automaticallyRecovered.has(deployment.serviceId)) continue; const current = affectedByServer.get(deployment.serverId) ?? { serverName: deployment.serverName, serverIp: @@ -118,7 +391,7 @@ export async function checkAndRecoverStaleServers( const markedOffline = await db .update(servers) - .set({ status: "offline" }) + .set({ status: "offline", onlineSince: null }) .where(and(...conditions)) .returning({ id: servers.id, @@ -146,9 +419,7 @@ export async function checkAndRecoverStaleServers( }); } - triggerRecoveryForOfflineServers(offlineIds).catch((error) => { - console.error("[scheduler] recovery failed:", error); - }); + await triggerRecoveryForOfflineServers(offlineIds); } export async function checkAndRunScheduledDeployments(): Promise { diff --git a/web/lib/service-config.ts b/web/lib/service-config.ts index 62627391..c083d41c 100644 --- a/web/lib/service-config.ts +++ b/web/lib/service-config.ts @@ -50,6 +50,7 @@ export type ResourceLimitsConfig = { }; export type PlacementConfig = { + mode?: "manual" | "automatic"; replicas: number; }; @@ -110,6 +111,7 @@ export function buildCurrentConfig( resourceCpuLimit: number | null; resourceMemoryLimitMb: number | null; replicas: number; + placementMode?: "manual" | "automatic" | null; stateful?: boolean | null; serverlessEnabled?: boolean | null; serverlessSleepAfterSeconds?: number | null; @@ -129,13 +131,19 @@ export function buildCurrentConfig( ): DeployedConfig { const hasResourceLimits = service.resourceCpuLimit != null || service.resourceMemoryLimitMb != null; + const placementMode = service.placementMode ?? "manual"; + const replicaCount = + placementMode === "automatic" + ? service.replicas + : replicas.reduce((sum, replica) => sum + replica.count, 0); return { source, hostname: service.hostname ?? undefined, stateful: service.stateful ?? false, placement: { - replicas: replicas.reduce((sum, r) => sum + r.count, 0), + mode: placementMode, + replicas: replicaCount, }, replicas: replicas.map((r) => ({ serverId: r.serverId, @@ -373,37 +381,62 @@ export function diffConfigs( }); } - const deployedReplicasMap = new Map( - (deployed.replicas || []).map((r) => [r.serverId, r]), - ); - const currentReplicasMap = new Map( - (current.replicas || []).map((r) => [r.serverId, r]), - ); - - for (const [serverId, currentReplica] of currentReplicasMap) { - const deployedReplica = deployedReplicasMap.get(serverId); - if (!deployedReplica) { - changes.push({ - field: `${currentReplica.serverName} replicas`, - from: "0", - to: String(currentReplica.count), - }); - } else if (deployedReplica.count !== currentReplica.count) { + const deployedPlacementMode = deployed.placement?.mode ?? "manual"; + const currentPlacementMode = current.placement?.mode ?? "manual"; + if (deployedPlacementMode !== currentPlacementMode) { + changes.push({ + field: "Placement mode", + from: deployedPlacementMode === "automatic" ? "Automatic" : "Manual", + to: currentPlacementMode === "automatic" ? "Automatic" : "Manual", + }); + } + if ( + deployedPlacementMode === "automatic" && + currentPlacementMode === "automatic" + ) { + if (deployed.placement?.replicas !== current.placement?.replicas) { changes.push({ - field: `${currentReplica.serverName} replicas`, - from: String(deployedReplica.count), - to: String(currentReplica.count), + field: "Desired replicas", + from: String(deployed.placement?.replicas ?? 0), + to: String(current.placement?.replicas ?? 0), }); } - } + } else if ( + deployedPlacementMode === "manual" && + currentPlacementMode === "manual" + ) { + const deployedReplicasMap = new Map( + (deployed.replicas || []).map((replica) => [replica.serverId, replica]), + ); + const currentReplicasMap = new Map( + (current.replicas || []).map((replica) => [replica.serverId, replica]), + ); + + for (const [serverId, currentReplica] of currentReplicasMap) { + const deployedReplica = deployedReplicasMap.get(serverId); + if (!deployedReplica) { + changes.push({ + field: `${currentReplica.serverName} replicas`, + from: "0", + to: String(currentReplica.count), + }); + } else if (deployedReplica.count !== currentReplica.count) { + changes.push({ + field: `${currentReplica.serverName} replicas`, + from: String(deployedReplica.count), + to: String(currentReplica.count), + }); + } + } - for (const [serverId, deployedReplica] of deployedReplicasMap) { - if (!currentReplicasMap.has(serverId)) { - changes.push({ - field: `${deployedReplica.serverName} replicas`, - from: String(deployedReplica.count), - to: "0 (removed)", - }); + for (const [serverId, deployedReplica] of deployedReplicasMap) { + if (!currentReplicasMap.has(serverId)) { + changes.push({ + field: `${deployedReplica.serverName} replicas`, + from: String(deployedReplica.count), + to: "0 (removed)", + }); + } } } @@ -645,6 +678,13 @@ export function revisionSpecToDeployedConfig( specification: ServiceRevisionSpec, serverNames: Record, ): DeployedConfig { + const replicaCount = + specification.placement.mode === "automatic" + ? specification.placement.replicas + : specification.placements.reduce( + (sum, placement) => sum + placement.count, + 0, + ); return { source: specification.source.type === "github" @@ -658,10 +698,8 @@ export function revisionSpecToDeployedConfig( hostname: specification.hostname, stateful: specification.stateful, placement: { - replicas: specification.placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ), + mode: specification.placement.mode, + replicas: replicaCount, }, replicas: specification.placements.map((placement) => ({ serverId: placement.serverId, diff --git a/web/lib/service-revision-changes.ts b/web/lib/service-revision-changes.ts index 5e3cd41f..8ab83abd 100644 --- a/web/lib/service-revision-changes.ts +++ b/web/lib/service-revision-changes.ts @@ -7,8 +7,7 @@ import type { ServiceRevisionSpec, } from "@/lib/service-revision-spec"; -const serviceRevisionSpecSchema = z.strictObject({ - schemaVersion: z.literal(2), +const serviceRevisionSpecFields = { image: z.string(), source: z.discriminatedUnion("type", [ z.strictObject({ type: z.literal("image"), image: z.string() }), @@ -72,7 +71,41 @@ const serviceRevisionSpecSchema = z.strictObject({ volumes: z.array( z.strictObject({ name: z.string(), containerPath: z.string() }), ), +}; + +const serviceRevisionSpecV2Schema = z.strictObject({ + schemaVersion: z.literal(2), + ...serviceRevisionSpecFields, }); +const serviceRevisionSpecSchema = z + .strictObject({ + schemaVersion: z.literal(3), + placement: z.discriminatedUnion("mode", [ + z.strictObject({ mode: z.literal("manual") }), + z.strictObject({ + mode: z.literal("automatic"), + replicas: z.number().int().min(1).max(10), + }), + ]), + ...serviceRevisionSpecFields, + }) + .superRefine((spec, context) => { + if (spec.placement.mode === "automatic" && spec.placements.length > 0) + context.addIssue({ + code: "custom", + message: "Automatic placement snapshots cannot contain placements", + }); + if (spec.placement.mode === "automatic" && spec.volumes.length > 0) + context.addIssue({ + code: "custom", + message: "Services with volumes cannot use automatic placement", + }); + if (spec.stateful && spec.placement.mode === "automatic") + context.addIssue({ + code: "custom", + message: "Stateful services cannot use automatic placement", + }); + }); export type ServiceRevisionChange = { field: string; @@ -102,7 +135,12 @@ export type ServiceRevisionChangelogResponse = { }; export function parseServiceRevisionSpec(value: unknown): ServiceRevisionSpec { - return serviceRevisionSpecSchema.parse(value); + const version = (value as { schemaVersion?: unknown } | null)?.schemaVersion; + if (version === 2) { + const legacy = serviceRevisionSpecV2Schema.parse(value); + return { ...legacy, schemaVersion: 3, placement: { mode: "manual" } }; + } + return serviceRevisionSpecSchema.parse(value) as ServiceRevisionSpec; } function compareStrings(a: string, b: string): number { @@ -249,25 +287,44 @@ export function diffServiceRevisionSpecs( : `${current.resourceLimits.memoryMb} MB`, ); - const previousPlacements = new Map( - previous.placements.map((placement) => [placement.serverId, placement]), - ); - const currentPlacements = new Map( - current.placements.map((placement) => [placement.serverId, placement]), + add( + "Placement mode", + previous.placement.mode === "automatic" ? "Automatic" : "Manual", + current.placement.mode === "automatic" ? "Automatic" : "Manual", ); - for (const serverId of [ - ...new Set([...previousPlacements.keys(), ...currentPlacements.keys()]), - ].sort(compareStrings)) { - const before = previousPlacements.get(serverId); - const after = currentPlacements.get(serverId); - const serverName = serverNames.get(serverId)?.trim(); + if ( + previous.placement.mode === "automatic" && + current.placement.mode === "automatic" + ) { add( - serverName - ? `${serverName} replicas` - : `Deleted server (${serverId.slice(0, 8)}) replicas`, - before ? `${before.count} replicas` : "(none)", - after ? `${after.count} replicas` : "(removed)", + "Desired replicas", + String(previous.placement.replicas), + String(current.placement.replicas), + ); + } else if ( + previous.placement.mode === "manual" && + current.placement.mode === "manual" + ) { + const previousPlacements = new Map( + previous.placements.map((placement) => [placement.serverId, placement]), ); + const currentPlacements = new Map( + current.placements.map((placement) => [placement.serverId, placement]), + ); + for (const serverId of [ + ...new Set([...previousPlacements.keys(), ...currentPlacements.keys()]), + ].sort(compareStrings)) { + const before = previousPlacements.get(serverId); + const after = currentPlacements.get(serverId); + const serverName = serverNames.get(serverId)?.trim(); + add( + serverName + ? `${serverName} replicas` + : `Deleted server (${serverId.slice(0, 8)}) replicas`, + before ? `${before.count} replicas` : "(none)", + after ? `${after.count} replicas` : "(removed)", + ); + } } const previousPorts = new Map( diff --git a/web/lib/service-revision-spec.ts b/web/lib/service-revision-spec.ts index 3e44c3e7..f400f348 100644 --- a/web/lib/service-revision-spec.ts +++ b/web/lib/service-revision-spec.ts @@ -1,4 +1,4 @@ -export const SERVICE_REVISION_SCHEMA_VERSION = 2 as const; +export const SERVICE_REVISION_SCHEMA_VERSION = 3 as const; export function getDefaultServiceHostname(name: string): string { return name @@ -20,6 +20,10 @@ export type ServiceRevisionPlacement = { count: number; }; +export type ServiceRevisionPlacementIntent = + | { mode: "manual" } + | { mode: "automatic"; replicas: number }; + export type ServiceRevisionPort = { containerPort: number; isPublic: boolean; @@ -74,6 +78,7 @@ export type ServiceRevisionSpec = { cpuCores: number | null; memoryMb: number | null; }; + placement: ServiceRevisionPlacementIntent; placements: ServiceRevisionPlacement[]; ports: ServiceRevisionPort[]; secrets: ServiceRevisionSecret[]; @@ -97,6 +102,8 @@ export type ServiceRevisionDraft = { startCommand: string | null; resourceCpuLimit: number | null; resourceMemoryLimitMb: number | null; + placementMode?: "manual" | "automatic" | null; + replicas?: number; }; placements: Array<{ serverId: string; count: number }>; ports: Array<{ @@ -125,10 +132,13 @@ function validateServiceRevisionSpec( specification: ServiceRevisionSpec, allowNoPlacements: boolean, ) { - const totalReplicas = specification.placements.reduce( - (sum, placement) => sum + placement.count, - 0, - ); + const totalReplicas = + specification.placement.mode === "automatic" + ? specification.placement.replicas + : specification.placements.reduce( + (sum, placement) => sum + placement.count, + 0, + ); if (totalReplicas < 1 && !allowNoPlacements) { throw new Error("At least one replica is required"); @@ -136,6 +146,21 @@ function validateServiceRevisionSpec( if (totalReplicas > 10) { throw new Error("Maximum 10 replicas allowed"); } + if ( + specification.placement.mode === "automatic" && + specification.placements.length + ) { + throw new Error("Automatic placement snapshots cannot contain placements"); + } + if ( + specification.placement.mode === "automatic" && + specification.volumes.length > 0 + ) { + throw new Error("Services with volumes cannot use automatic placement"); + } + if (specification.stateful && specification.placement.mode === "automatic") { + throw new Error("Stateful services cannot use automatic placement"); + } if (specification.stateful && totalReplicas !== 1) { throw new Error("Stateful services can only have exactly 1 replica"); } @@ -195,7 +220,11 @@ export function buildServiceRevisionSpec( cpuCores: service.resourceCpuLimit, memoryMb: service.resourceMemoryLimitMb, }, - placements: draft.placements + placement: + service.placementMode === "automatic" + ? { mode: "automatic", replicas: service.replicas ?? 1 } + : { mode: "manual" }, + placements: (service.placementMode === "automatic" ? [] : draft.placements) .filter((placement) => placement.count > 0) .map((placement) => ({ serverId: placement.serverId, diff --git a/web/lib/service-revisions.ts b/web/lib/service-revisions.ts index cc05aa31..6426c96f 100644 --- a/web/lib/service-revisions.ts +++ b/web/lib/service-revisions.ts @@ -1,7 +1,8 @@ import { randomUUID } from "node:crypto"; -import { and, eq, isNull } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; import { db } from "@/db"; import { + deployments, githubRepos, rollouts, secrets, @@ -16,7 +17,6 @@ import type { ServiceRevisionActor } from "@/lib/service-revision-actor"; import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; import { buildServiceRevisionSpec, - SERVICE_REVISION_SCHEMA_VERSION, type ServiceRevisionSource, type ServiceRevisionSpec, type ServiceRevisionSpecOverrides, @@ -186,76 +186,74 @@ export async function createGitHubBuildServiceRevision(input: { expectedBranch: string; actor: ServiceRevisionActor | null; }) { - return db.transaction( - async (tx) => { - const existing = await tx - .select() - .from(serviceRevisions) - .where(eq(serviceRevisions.id, input.id)) - .then((rows) => rows[0]); - if (existing) { - assertMatchingGitHubBuildRevision(existing, input); - return existing; - } + return db.transaction(async (tx) => { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtext(${input.serviceId}))`, + ); + const existing = await tx + .select() + .from(serviceRevisions) + .where(eq(serviceRevisions.id, input.id)) + .then((rows) => rows[0]); + if (existing) { + assertMatchingGitHubBuildRevision(existing, input); + return existing; + } - const [service, repo] = await Promise.all([ - tx - .select() - .from(services) - .where( - and(eq(services.id, input.serviceId), isNull(services.deletedAt)), - ) - .then((rows) => rows[0]), - tx - .select() - .from(githubRepos) - .where(eq(githubRepos.serviceId, input.serviceId)) - .then((rows) => rows[0]), - ]); - if (!service || service.sourceType !== "github") { - throw new Error("Active GitHub service not found"); - } + const [service, repo] = await Promise.all([ + tx + .select() + .from(services) + .where( + and(eq(services.id, input.serviceId), isNull(services.deletedAt)), + ) + .then((rows) => rows[0]), + tx + .select() + .from(githubRepos) + .where(eq(githubRepos.serviceId, input.serviceId)) + .then((rows) => rows[0]), + ]); + if (!service || service.sourceType !== "github") { + throw new Error("Active GitHub service not found"); + } - const currentSource = resolvePersistedSourceFromRows(service, repo); - if ( - currentSource.type !== "github" || - !currentSource.repository || - currentSource.repository !== input.expectedRepository || - currentSource.branch !== input.expectedBranch - ) { - throw new Error( - "GitHub source changed while resolving the build commit", - ); - } + const currentSource = resolvePersistedSourceFromRows(service, repo); + if ( + currentSource.type !== "github" || + !currentSource.repository || + currentSource.repository !== input.expectedRepository || + currentSource.branch !== input.expectedBranch + ) { + throw new Error("GitHub source changed while resolving the build commit"); + } - const source: ServiceRevisionSource = { - type: "github", - repository: currentSource.repository, - repositoryId: repo?.repoId ?? null, - branch: currentSource.branch, - commitSha: input.commitSha, - rootDir: currentSource.rootDir?.trim() || null, - authentication: repo - ? { - type: "github_app", - installationId: repo.installationId, - } - : { type: "anonymous" }, - }; + const source: ServiceRevisionSource = { + type: "github", + repository: currentSource.repository, + repositoryId: repo?.repoId ?? null, + branch: currentSource.branch, + commitSha: input.commitSha, + rootDir: currentSource.rootDir?.trim() || null, + authentication: repo + ? { + type: "github_app", + installationId: repo.installationId, + } + : { type: "anonymous" }, + }; - return createServiceRevisionSnapshot(tx, { - id: input.id, - serviceId: input.serviceId, - actor: input.actor, - overrides: { - image: input.image, - source, - allowNoPlacements: true, - }, - }); - }, - { isolationLevel: "repeatable read" }, - ); + return createServiceRevisionSnapshot(tx, { + id: input.id, + serviceId: input.serviceId, + actor: input.actor, + overrides: { + image: input.image, + source, + allowNoPlacements: true, + }, + }); + }); } export async function cloneGitHubBuildServiceRevision(input: { @@ -320,6 +318,9 @@ export async function createRolloutWithServiceRevision( ) { return db.transaction( async (tx) => { + await tx.execute( + sql`select pg_advisory_xact_lock(hashtext(${serviceId}))`, + ); let overrides: ServiceRevisionSpecOverrides | undefined; if (runtimeBaseRevisionId) { const baseRevision = await tx @@ -366,12 +367,72 @@ export async function createRolloutWithServiceRevision( ); } +/** Clone the deployed specification, never mutable service configuration. */ +export async function cloneActiveRevisionAndQueueSystemRollout( + serviceId: string, + sourceRevisionId?: string, +) { + return db.transaction(async (tx) => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${serviceId}))`); + const pending = await tx + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .limit(1) + .then((rows) => rows[0]); + if (pending) return { rolloutId: pending.id, created: false }; + const active = await tx + .select({ specification: serviceRevisions.specification }) + .from(serviceRevisions) + .innerJoin( + deployments, + eq(deployments.serviceRevisionId, serviceRevisions.id), + ) + .where( + and( + eq(serviceRevisions.serviceId, serviceId), + ...(sourceRevisionId + ? [eq(serviceRevisions.id, sourceRevisionId)] + : []), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + ), + ) + .orderBy(desc(deployments.createdAt), desc(deployments.id)) + .limit(1) + .then((rows) => rows[0]); + if (!active) throw new Error("Service has no active revision"); + const revisionId = randomUUID(); + await tx.insert(serviceRevisions).values({ + id: revisionId, + serviceId, + specification: active.specification, + actor: { type: "system" }, + }); + const rolloutId = randomUUID(); + await tx.insert(rollouts).values({ + id: rolloutId, + serviceId, + serviceRevisionId: revisionId, + status: "queued", + currentStage: "queued", + }); + return { rolloutId, created: true }; + }); +} + export async function createRolloutForServiceRevision( serviceId: string, serviceRevisionId: string, artifactImageUri: string, ) { return db.transaction(async (tx) => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${serviceId}))`); const [revision, activeService] = await Promise.all([ tx .select() @@ -398,7 +459,10 @@ export async function createRolloutForServiceRevision( if (specification.image !== artifactImageUri) { throw new Error("Built artifact does not match the service revision"); } - if (!specification.placements.some((placement) => placement.count > 0)) { + if ( + specification.placement.mode === "manual" && + !specification.placements.some((placement) => placement.count > 0) + ) { return { rolloutId: null, revision, created: false }; } @@ -441,12 +505,8 @@ export async function getRolloutServiceRevision(rolloutId: string) { .then((rows) => rows[0]); if (!result) throw new Error("Rollout revision not found"); - if ( - result.revision.specification.schemaVersion !== - SERVICE_REVISION_SCHEMA_VERSION - ) { - throw new Error("Unsupported service revision schema version"); - } - - return result.revision; + return { + ...result.revision, + specification: parseServiceRevisionSpec(result.revision.specification), + }; } diff --git a/web/tests/autoplacement.test.ts b/web/tests/autoplacement.test.ts new file mode 100644 index 00000000..33856ed5 --- /dev/null +++ b/web/tests/autoplacement.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { distributeReplicas } from "@/lib/inngest/functions/rollout-helpers"; +import { parseServiceRevisionSpec } from "@/lib/service-revision-changes"; + +describe("automatic placement distribution", () => { + it.each([ + [1, [1, 0, 0]], + [3, [1, 1, 1]], + [5, [2, 2, 1]], + ])("distributes %i replicas over three servers", (replicas, counts) => { + const result = distributeReplicas(["c", "a", "b"], replicas); + expect(result.map((placement) => placement.serverId)).toEqual( + ["a", "b", "c"].slice(0, result.length), + ); + expect( + ["a", "b", "c"].map( + (id) => result.find((p) => p.serverId === id)?.replicas ?? 0, + ), + ).toEqual(counts); + }); + + it("stacks ten replicas deterministically on two servers", () => { + expect(distributeReplicas(["b", "a"], 10)).toEqual([ + { serverId: "a", replicas: 5 }, + { serverId: "b", replicas: 5 }, + ]); + }); +}); + +describe("persisted revision compatibility", () => { + it("normalizes v2 revisions to v3 manual intent", () => { + const parsed = parseServiceRevisionSpec({ + schemaVersion: 2, + image: "nginx", + source: { type: "image", image: "nginx" }, + hostname: "web", + stateful: false, + serverless: { + enabled: false, + sleepAfterSeconds: 300, + wakeTimeoutSeconds: 300, + }, + healthCheck: null, + startCommand: null, + resourceLimits: { cpuCores: null, memoryMb: null }, + placements: [{ serverId: "a", count: 1 }], + ports: [], + secrets: [], + volumes: [], + }); + expect(parsed.schemaVersion).toBe(3); + expect(parsed.placement).toEqual({ mode: "manual" }); + }); +}); diff --git a/web/tests/build-assignment.test.ts b/web/tests/build-assignment.test.ts index 7a18320f..fbe1d133 100644 --- a/web/tests/build-assignment.test.ts +++ b/web/tests/build-assignment.test.ts @@ -39,7 +39,8 @@ function specification( overrides: Partial = {}, ): ServiceRevisionSpec { return { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image: "registry/app:revision-1", source: { type: "image", image: "registry/app:revision-1" }, hostname: "app", diff --git a/web/tests/deploy-service-revision.test.ts b/web/tests/deploy-service-revision.test.ts index ef044af3..f13b2505 100644 --- a/web/tests/deploy-service-revision.test.ts +++ b/web/tests/deploy-service-revision.test.ts @@ -8,6 +8,7 @@ const mocks = vi.hoisted(() => ({ startMigrationInternal: vi.fn(), triggerBuildInternal: vi.fn(), send: vi.fn(), + updateWhere: vi.fn(), createRolloutCreated: vi.fn((data, options) => ({ name: "rollout/created", data, @@ -17,6 +18,9 @@ const mocks = vi.hoisted(() => ({ vi.mock("@/db", () => ({ db: { + update: vi.fn(() => ({ + set: vi.fn(() => ({ where: mocks.updateWhere })), + })), select: vi.fn(() => ({ from: vi.fn(() => ({ where: vi.fn(() => Promise.resolve(mocks.replicaRows)), @@ -129,6 +133,7 @@ describe("revision rollout idempotency", () => { expect(mocks.send.mock.calls[0]?.[0].id).toBe( mocks.send.mock.calls[1]?.[0].id, ); + expect(mocks.updateWhere).toHaveBeenCalledTimes(1); }); it("bases a GitHub redeployment on an explicit runtime revision", async () => { diff --git a/web/tests/public-api-configuration.test.ts b/web/tests/public-api-configuration.test.ts index a255c6f6..f17dce62 100644 --- a/web/tests/public-api-configuration.test.ts +++ b/web/tests/public-api-configuration.test.ts @@ -89,4 +89,35 @@ describe("public API configuration state", () => { expect(configuration.hasPendingChanges).toBe(false); expect(configuration.changes).toEqual([]); }); + + it("serializes automatic placement from desired service replicas", async () => { + mocks.rows.push([], [], [], [], []); + const configuration = await safeConfiguration({ + id: "service-1", + name: "Automatic", + sourceType: "image", + image: "nginx", + hostname: null, + stateful: false, + placementMode: "automatic", + replicas: 4, + healthCheckCmd: null, + startCommand: null, + resourceCpuLimit: null, + resourceMemoryLimitMb: null, + serverlessEnabled: false, + serverlessSleepAfterSeconds: 300, + serverlessWakeTimeoutSeconds: 300, + deploymentSchedule: null, + backupEnabled: false, + backupSchedule: null, + } as never); + + expect(configuration.current.placement).toEqual({ + mode: "automatic", + replicas: 4, + }); + expect(configuration.current.replicas).toBe(4); + expect(configuration.current.placements).toEqual([]); + }); }); diff --git a/web/tests/public-api-source.test.ts b/web/tests/public-api-source.test.ts index a31698e4..71189f94 100644 --- a/web/tests/public-api-source.test.ts +++ b/web/tests/public-api-source.test.ts @@ -113,3 +113,37 @@ describe("public API GitHub sources", () => { expect(cleared).toHaveProperty("rootDir", null); }); }); + +describe("public API placement schema", () => { + it.each([ + { mode: "automatic", replicas: 3 }, + { mode: "manual", placements: [{ serverId: "server-1", count: 2 }] }, + ])("accepts valid placement intent", (placement) => { + expect(configurationPatchSchema.safeParse({ placement }).success).toBe( + true, + ); + }); + + it.each([ + { mode: "automatic", replicas: 0 }, + { mode: "manual", placements: [] }, + { + mode: "manual", + placements: [ + { serverId: "a", count: 6 }, + { serverId: "b", count: 5 }, + ], + }, + { + mode: "manual", + placements: [ + { serverId: "a", count: 1 }, + { serverId: "a", count: 1 }, + ], + }, + ])("rejects invalid placement intent", (placement) => { + expect(configurationPatchSchema.safeParse({ placement }).success).toBe( + false, + ); + }); +}); diff --git a/web/tests/service-config.test.ts b/web/tests/service-config.test.ts index f6f65794..dad82e80 100644 --- a/web/tests/service-config.test.ts +++ b/web/tests/service-config.test.ts @@ -44,7 +44,8 @@ describe("service config", () => { it("converts an immutable revision for pending-change comparisons", () => { const config = revisionSpecToDeployedConfig( { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image: "nginx", source: { type: "image", image: "nginx" }, hostname: "api", @@ -114,7 +115,8 @@ describe("service config", () => { ] as const) { const active = revisionSpecToDeployedConfig( { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image, source: { ...currentSource, diff --git a/web/tests/service-revision-build.test.ts b/web/tests/service-revision-build.test.ts index 3539c9d5..8a61ddd5 100644 --- a/web/tests/service-revision-build.test.ts +++ b/web/tests/service-revision-build.test.ts @@ -35,6 +35,7 @@ const mocks = vi.hoisted(() => { return query; } const tx = { + execute: vi.fn(() => Promise.resolve()), select: vi.fn(() => selectQuery(selectResults.shift() ?? [])), insert: vi.fn(() => insertQuery()), }; @@ -64,7 +65,8 @@ import { function sourceSpecification(): ServiceRevisionSpec { return { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image: "registry.test/project-1/service-1:revision-original", source: { type: "github", @@ -162,6 +164,7 @@ describe("GitHub build service revisions", () => { actor: { type: "system" }, }), ).rejects.toThrow("Service revision idempotency conflict"); + expect(mocks.tx.execute).toHaveBeenCalledTimes(1); expect(mocks.tx.insert).not.toHaveBeenCalled(); }); diff --git a/web/tests/service-revision-changes.test.ts b/web/tests/service-revision-changes.test.ts index 84dd722c..1b068782 100644 --- a/web/tests/service-revision-changes.test.ts +++ b/web/tests/service-revision-changes.test.ts @@ -4,7 +4,8 @@ import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; function spec(): ServiceRevisionSpec { return { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image: "app:v1", source: { type: "image", image: "app:v1" }, hostname: "app", @@ -138,6 +139,23 @@ describe("diffServiceRevisionSpecs", () => { }); }); + it("reports automatic placement intent without transient server assignments", () => { + const previous = spec(); + previous.placement = { mode: "automatic", replicas: 2 }; + previous.placements = []; + const current = structuredClone(previous); + current.placement = { mode: "automatic", replicas: 4 }; + + expect(diffServiceRevisionSpecs(previous, current)).toEqual([ + { field: "Desired replicas", from: "2", to: "4" }, + ]); + + const manual = spec(); + expect(diffServiceRevisionSpecs(manual, previous)).toEqual([ + { field: "Placement mode", from: "Manual", to: "Automatic" }, + ]); + }); + it("never exposes secret ciphertext while detecting additions, updates, and removals", () => { const previous = spec(); previous.secrets.push({ diff --git a/web/tests/service-revision-spec.test.ts b/web/tests/service-revision-spec.test.ts index 4fd3add4..70d9aaa7 100644 --- a/web/tests/service-revision-spec.test.ts +++ b/web/tests/service-revision-spec.test.ts @@ -120,7 +120,7 @@ describe("service revision specification", () => { }); expect(spec).toMatchObject({ - schemaVersion: 2, + schemaVersion: 3, image: "registry.test/project/service:revision-1", source: { type: "github", @@ -141,6 +141,36 @@ describe("service revision specification", () => { ).not.toThrow(); }); + it("snapshots automatic placement intent without resolved placements", () => { + const input = draft({ volumes: [] }); + input.service.placementMode = "automatic"; + input.service.replicas = 4; + + expect(buildServiceRevisionSpec(input)).toMatchObject({ + placement: { mode: "automatic", replicas: 4 }, + placements: [], + }); + }); + + it("rejects automatic placement for stateful and volume-backed services", () => { + const stateful = draft({ volumes: [] }); + stateful.service.stateful = true; + stateful.service.placementMode = "automatic"; + stateful.service.replicas = 1; + + expect(() => buildServiceRevisionSpec(stateful)).toThrow( + "Stateful services cannot use automatic placement", + ); + + const volumeBacked = draft(); + volumeBacked.service.placementMode = "automatic"; + volumeBacked.service.replicas = 1; + + expect(() => buildServiceRevisionSpec(volumeBacked)).toThrow( + "Services with volumes cannot use automatic placement", + ); + }); + it("rejects serverless revisions without a public HTTP port and domain", () => { const input = draft({ ports: [ diff --git a/web/tests/service-revisions-route.test.ts b/web/tests/service-revisions-route.test.ts index 43f5955e..673da3f0 100644 --- a/web/tests/service-revisions-route.test.ts +++ b/web/tests/service-revisions-route.test.ts @@ -40,7 +40,8 @@ function revisionSpec( encryptedValue = "cipher", ): ServiceRevisionSpec { return { - schemaVersion: 2, + schemaVersion: 3, + placement: { mode: "manual" }, image, source: { type: "image", image }, hostname: "app", From e656294aa0d46834c2b323f79027fde540cd01d4 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:53:59 +1000 Subject: [PATCH 02/43] Harden automatic placement recovery --- deployment/README.md | 17 +- docs/api/public-api.mdx | 2 +- docs/architecture.mdx | 35 ++-- docs/index.mdx | 3 +- docs/services/domains.mdx | 17 +- docs/services/scaling.mdx | 20 +- proxy/README.md | 18 +- web/actions/projects.ts | 21 +- .../service/details/replicas-section.tsx | 19 +- .../service/details/serverless-section.tsx | 13 +- .../settings/edge-domain-settings.tsx | 20 +- web/db/schema.ts | 4 + web/lib/inngest/functions/crons.ts | 23 ++- web/lib/public-api.ts | 13 +- web/lib/scheduler.ts | 186 ++++++++++++------ web/lib/service-revision-changes.ts | 5 + web/lib/service-revision-spec.ts | 6 + web/tests/service-revision-changes.test.ts | 16 +- web/tests/service-revision-spec.test.ts | 11 +- 19 files changed, 304 insertions(+), 145 deletions(-) diff --git a/deployment/README.md b/deployment/README.md index 0a5720c5..7ffdcfd8 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -80,10 +80,16 @@ After signing in as an admin, configure **Edge Domain** under public edge traffic, including HTTP/HTTPS custom domains and direct TCP/UDP connection strings. -Configure this hostname in your DNS provider to resolve to every proxy server's -public IPv4 address. You can use direct `A` records or your own DNS routing, -load-balancing, or GeoDNS policy. Custom HTTP/HTTPS subdomains can use a `CNAME` -to the edge domain; apex domains can use `ALIAS` or `ANAME` where supported. +A stable external load balancer with active health checks is the ideal production +solution for proxy failure. Configure each proxy public IPv4 address as an origin, +then point the edge hostname to the load balancer's stable address. A direct `A` +record to one proxy has no ingress failover. Multiple proxy `A` records provide +best-effort DNS distribution, but cached answers may continue sending clients to +an offline proxy. + +Custom HTTP/HTTPS subdomains can use a `CNAME` to the edge domain; apex domains +can use `ALIAS` or `ANAME` where supported. Health-aware GeoDNS is an alternative, +but its failover time remains subject to DNS and client caching. Techulus Cloud only displays the required DNS configuration. It does not create or update DNS records. @@ -94,8 +100,7 @@ Configure **Automatic Subdomain Domain** under **Settings → Infrastructure** t offer generated domains in service Networking settings. Enter only the base domain, such as `apps.example.com`, without `*.` or a protocol. -Create wildcard DNS records for `*.apps.example.com` that resolve to every proxy -server public IPv4 address. +Create a wildcard `CNAME` for `*.apps.example.com` that points to the edge domain. ### Web Replicas diff --git a/docs/api/public-api.mdx b/docs/api/public-api.mdx index 136d0a01..70e9ce53 100644 --- a/docs/api/public-api.mdx +++ b/docs/api/public-api.mdx @@ -209,7 +209,7 @@ Use manual placement to choose exact servers: } ``` -Manual placement requires online servers with WireGuard configured. Serverless services require proxy servers. Automatic placement is not available for stateful or volume-backed services. The legacy top-level `replicas` field remains supported when placement is unchanged. It must match the existing manual placement. +Manual placement requires online servers with WireGuard configured. Serverless services require proxy servers. Automatic placement is not available for stateful, serverless, or volume-backed services. The legacy top-level `replicas` field remains supported when placement is unchanged. It must match the existing manual placement. The API only manages stateless services with HTTP ports. Existing volumes, stateful mode, TCP or UDP ports, TLS passthrough, or invalid resource limits return a conflict with an actionable code. diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 0bc4d385..a7de10d8 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -210,8 +210,8 @@ Worker nodes do not run Traefik. Public HTTP services can be configured as serverless. Serverless scale-to-zero is proxy-local: deployments placed on proxy nodes may sleep after an idle period, -then wake on the next public request handled by that proxy. Deployments placed on -worker nodes stay always on and remain routable. +then wake on the next public request handled by that proxy. Serverless services +use manual placement on proxy nodes. Serverless uses the same declarative expected-state model as normal deployments, but proxy agents own the local lifecycle decision: @@ -220,8 +220,6 @@ deployments, but proxy agents own the local lifecycle decision: `runtimeDesiredState: "stopped"` and `observedPhase: "sleeping"`. - Expected state advertises proxy-hosted sleeping containers with `desiredState: "stopped"` so normal reconciliation does not restart them. -- Expected state advertises worker-hosted deployments with - `desiredState: "running"` even when the service is serverless-enabled. - The proxy gateway wakes local sleeping deployments from expected-state metadata and reports `wake_started` through the next status report. - The proxy agent reports local sleep with a `sleep` status transition after it @@ -248,18 +246,15 @@ sequenceDiagram Proxy Traefik routes point to the local wake gateway only on proxy nodes that host a local proxy deployment for that serverless service. Non-owner proxies do -not emit a public HTTP route for that service, even when always-on worker -upstreams exist. Worker-only serverless services use normal direct routes because -there is nothing local to wake. +not emit a public HTTP route for that service. Public DNS or external load balancers must therefore send serverless traffic only to proxy nodes that own a local proxy replica for that service. Cross-proxy wake coordination is intentionally out of scope. The wake gateway keeps the incoming request open while it starts local proxy -replicas, unless an always-on worker upstream is already ready. Queued requests -resume when one upstream is ready. If no upstream becomes ready and no local wake -is still in progress, the gateway returns a 503. +replicas. Queued requests resume when one upstream is ready. If no upstream +becomes ready and no local wake is still in progress, the gateway returns a 503. Sleep is driven by the proxy gateway's in-memory activity timer. The control plane does not scan request activity and does not enqueue sleep work. It accepts @@ -273,11 +268,13 @@ deployments. ### Multiple Proxy Nodes -The platform supports geographically distributed proxy nodes with proximity steering: +The platform supports multiple proxy nodes behind a shared edge hostname. A +stable external load balancer with active health checks is the ideal production +solution for proxy failure: -- Users point custom domains to a single GeoDNS-managed hostname. -- GeoDNS routes clients to the nearest healthy proxy. -- Health checks fail over automatically when a proxy becomes unavailable. +- Users point custom domains to one shared edge hostname. +- The load balancer sends new connections only to healthy proxy origins. +- Workload placement changes proxy routes without changing public DNS. - All proxies share the same TLS certificates from the control plane. Example: @@ -287,12 +284,16 @@ Proxy US: 1.2.3.4 Proxy EU: 5.6.7.8 Proxy SYD: 9.10.11.12 -GeoDNS: +External load balancer: example.com -> lb.techulus.cloud - -> route client to nearest proxy - -> fail over when a proxy is unhealthy + -> actively health-check proxy origins + -> stop new traffic to an unhealthy proxy ``` +Health-aware GeoDNS can provide proximity steering, but recovery remains subject +to resolver and client caching. Plain multiple A records are best-effort traffic +distribution, not reliable proxy failover. + ### Proximity-Aware Load Balancing Within a proxy node, traffic is distributed using weighted round-robin: diff --git a/docs/index.mdx b/docs/index.mdx index c42edd84..df09e725 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -25,7 +25,8 @@ All communication between nodes happens over an encrypted **WireGuard mesh netwo - **Persistent volumes** — attach named local volumes for stateful workloads with scheduled backups to S3-compatible storage. Replicated storage and HA failover are not yet supported. - **Service discovery** — containers resolve each other by name using `.internal` domains. - **Multi-environment** — run production, staging, and dev within the same project. -- **GeoDNS** — route users to the nearest proxy node with automatic failover. +- **External edge compatibility** — use a stable load balancer with active health + checks, or health-aware GeoDNS when DNS caching tradeoffs are acceptable. - **TCP/UDP proxy** — expose non-HTTP services like game servers or custom protocols. ## Next Steps diff --git a/docs/services/domains.mdx b/docs/services/domains.mdx index 197be2cc..9ec13ef4 100644 --- a/docs/services/domains.mdx +++ b/docs/services/domains.mdx @@ -7,7 +7,7 @@ description: "Bind custom domains with automatic HTTPS." Custom domains are bound to a specific service port. When you add a domain: -1. Point your domain's DNS to a proxy node's IP address (or your GeoDNS hostname). +1. Point your domain's DNS to your edge hostname. 2. Add the domain in the service port settings. 3. The platform automatically provisions a TLS certificate via Let's Encrypt. @@ -20,9 +20,9 @@ offer automatic domains in service networking settings. For example, with the setting configured as `apps.example.com`, a service whose private hostname is `project-api-production` can use `project-api-production.apps.example.com`. -Create a wildcard DNS record for `*.apps.example.com` that points to all proxy -servers. The setting contains only the base domain, without `*.` or a protocol. -Custom domains remain available alongside automatic domains. +Create a wildcard `CNAME` record for `*.apps.example.com` that points to your +edge hostname. The setting contains only the base domain, without `*.` or a +protocol. Custom domains remain available alongside automatic domains. ## TLS Certificates @@ -37,7 +37,14 @@ Certificates are: When using multiple proxy nodes for geographic distribution, all proxies share the same TLS certificates from the control plane. -Set up a GeoDNS hostname that routes clients to the nearest healthy proxy, then point your custom domains to that hostname. See the [Architecture](/architecture#multiple-proxy-nodes) page for details. +A stable external load balancer with active health checks is the ideal production +solution for proxy failure. Configure every proxy as an origin, point the edge +hostname to the load balancer, and point custom domains to the edge hostname. + +Health-aware GeoDNS is an alternative, but failover remains subject to DNS and +client caching. Plain multiple A records provide best-effort distribution and do +not guarantee that clients avoid an offline proxy. See the +[Architecture](/architecture#multiple-proxy-nodes) page for details. ## Protocols diff --git a/docs/services/scaling.mdx b/docs/services/scaling.mdx index 387d0115..9f46fb99 100644 --- a/docs/services/scaling.mdx +++ b/docs/services/scaling.mdx @@ -17,11 +17,6 @@ and stops the local container. The next public HTTP request wakes local proxy-hosted deployments and is held until an upstream is ready or the wake timeout is reached. -Worker-hosted deployments do not sleep. If a serverless-enabled service has only -worker replicas, it stays always on and routes directly. If a service has both -proxy and worker replicas, proxy replicas may sleep while worker replicas remain -routable. - For services with proxy-hosted serverless replicas, public traffic must be sent only to proxy nodes that host a local proxy replica for that service. Non-owner proxies do not emit a public HTTP route for the service, so DNS or the external @@ -36,15 +31,20 @@ Serverless settings are configured per service: | Wake timeout | `300s` | Maximum time the wake gateway waits for ready upstreams | A cold wake starts the sleeping local proxy replicas for that host. Held -requests resume when one upstream is ready. If a worker upstream is already -ready, the gateway can serve it immediately while local proxy replicas wake in -the background. +requests resume when one upstream is ready. Serverless services require at least one configured replica. ## Placement -Services use manual placement. Select the target servers and replica counts explicitly before deploying. +Stateless services support automatic and manual placement. Automatic placement +stores the desired replica count and distributes replicas across stable, +configured nodes during rollout. Manual placement selects exact target servers +and replica counts. + +Serverless services currently require manual placement on proxy nodes. Automatic +serverless placement will remain unavailable until every healthy proxy can route +requests through the owning proxy's wake gateway. ## Server Pinning @@ -61,6 +61,6 @@ You can also manually lock any service to a specific server by setting the locke - Stateful services do not automatically fail over to another server. - Maximum 10 replicas per service. - Serverless scaling requires a public HTTP service domain. -- Sleep and wake are proxy-local. Worker replicas are intentionally always on. +- Sleep and wake are proxy-local; serverless replicas must be placed on proxy nodes. - Serverless traffic must be routed only to proxy nodes that own a local proxy replica for that service. - Proxy agents report sleep and wake transitions through normal status reports. diff --git a/proxy/README.md b/proxy/README.md index d9b4e631..8e216569 100644 --- a/proxy/README.md +++ b/proxy/README.md @@ -51,10 +51,20 @@ Access at `http://:8080/dashboard/` ## DNS Setup -Point your domain's DNS to the proxy node(s) public IP: - -- **Single proxy**: A record pointing to proxy IP -- **Multiple proxies**: Multiple A records for DNS round-robin, or use a load balancer +Point your domain at a stable edge address: + +- **Production**: use a stable external load balancer with active health checks, + and configure every proxy public IP as an origin +- **Single proxy**: an A record can point directly to the proxy, but there is no + ingress failover +- **Multiple A records**: provide best-effort distribution, not reliable failover; + clients may cache and continue using an offline proxy +- **Health-aware GeoDNS**: supported as an alternative, with failover still + subject to DNS and client caching + +A stable external load balancer with active health checks is the ideal production +solution for proxy failure. Workload rebalancing updates proxy routes; it does not +modify public DNS or remove failed proxies from external traffic. ## How It Works diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 41e96782..45e79701 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -920,6 +920,11 @@ export async function updateServiceServerlessSettings( } if (validated.enabled) { + if (service.placementMode === "automatic") { + throw new Error( + "Switch to manual placement before enabling serverless", + ); + } const publicHttpPorts = await tx .select({ id: servicePorts.id }) .from(servicePorts) @@ -947,20 +952,16 @@ export async function updateServiceServerlessSettings( .from(serviceReplicas) .innerJoin(servers, eq(serviceReplicas.serverId, servers.id)) .where(eq(serviceReplicas.serviceId, serviceId)); - const totalConfiguredReplicas = - service.placementMode === "automatic" - ? service.replicas - : configuredReplicas.reduce( - (total, replica) => total + replica.count, - 0, - ); + const totalConfiguredReplicas = configuredReplicas.reduce( + (total, replica) => total + replica.count, + 0, + ); if (totalConfiguredReplicas < 1) { throw new Error("Serverless services require at least one replica"); } if ( - service.placementMode === "manual" && configuredReplicas.some( (replica) => replica.count > 0 && !replica.serverIsProxy, ) @@ -1177,6 +1178,10 @@ export async function updateServiceConfig( if (!currentService) throw new Error("Service not found"); if (placement.mode === "automatic") { + if (currentService.serverlessEnabled) + throw new Error( + "Automatic placement is not supported for serverless services", + ); const volume = await tx .select({ id: serviceVolumes.id }) .from(serviceVolumes) diff --git a/web/components/service/details/replicas-section.tsx b/web/components/service/details/replicas-section.tsx index dfc8491b..2a1df621 100644 --- a/web/components/service/details/replicas-section.tsx +++ b/web/components/service/details/replicas-section.tsx @@ -53,7 +53,7 @@ export const ReplicasSection = memo(function ReplicasSection({ ); const [selectedServerId, setSelectedServerId] = useState(null); const [placementMode, setPlacementMode] = useState( - service.placementMode, + service.serverlessEnabled ? "manual" : service.placementMode, ); const [desiredReplicas, setDesiredReplicas] = useState(service.replicas); const [isEditing, setIsEditing] = useState(false); @@ -86,7 +86,9 @@ export const ReplicasSection = memo(function ReplicasSection({ } } setLocalReplicas(replicaMap); - setPlacementMode(service.placementMode); + setPlacementMode( + service.serverlessEnabled ? "manual" : service.placementMode, + ); setDesiredReplicas(service.replicas); } }, [ @@ -95,6 +97,7 @@ export const ReplicasSection = memo(function ReplicasSection({ service.stateful, service.lockedServerId, service.placementMode, + service.serverlessEnabled, service.replicas, isEditing, ]); @@ -202,6 +205,7 @@ export const ReplicasSection = memo(function ReplicasSection({ const handleModeChange = (mode: string) => { const nextMode = mode as PlacementMode; + if (nextMode === "automatic" && service.serverlessEnabled) return; if (nextMode === placementMode) return; setIsEditing(true); if (nextMode === "automatic") { @@ -380,9 +384,11 @@ export const ReplicasSection = memo(function ReplicasSection({
- - Automatic - + {!service.serverlessEnabled && ( + + Automatic + + )} Manual @@ -397,8 +403,7 @@ export const ReplicasSection = memo(function ReplicasSection({

The control plane distributes replicas evenly across healthy - {service.serverlessEnabled ? " proxy nodes" : " nodes"} and - moves them after failures. + nodes and moves them after failures.

replica.count > 0 && !replica.serverIsProxy, ); - const unavailableReason = !hasPublicHttpEndpoint - ? "Add a public HTTP port with a domain to enable serverless" - : hasWorkerReplica - ? "Serverless services can only be deployed to proxy nodes" - : null; + const unavailableReason = + service.placementMode === "automatic" + ? "Switch to manual placement before enabling serverless" + : !hasPublicHttpEndpoint + ? "Add a public HTTP port with a domain to enable serverless" + : hasWorkerReplica + ? "Serverless services can only be deployed to proxy nodes" + : null; const optionsDisabled = !!unavailableReason || isSaving; const parsed = useMemo( diff --git a/web/components/settings/edge-domain-settings.tsx b/web/components/settings/edge-domain-settings.tsx index 4368de33..110f6abf 100644 --- a/web/components/settings/edge-domain-settings.tsx +++ b/web/components/settings/edge-domain-settings.tsx @@ -114,10 +114,16 @@ export function EdgeDomainSettings({
- +

- Create one A record for each proxy IPv4 address using your DNS - provider and routing policy. + Configure these addresses as origins behind your external load + balancer. A stable external load balancer with active health + checks is the ideal production solution for proxy failure. +

+

+ A direct A record to one proxy has no ingress failover. Multiple A + records provide best-effort distribution, but clients may continue + using an offline proxy because of DNS caching.

{initial.hostname && ipv4Targets.length > 0 ? ( @@ -127,7 +133,7 @@ export function EdgeDomainSettings({ className="grid gap-1 px-3 py-2 text-sm sm:grid-cols-[4rem_1fr_1fr]" > - A + IP {initial.hostname} @@ -199,12 +205,14 @@ export function EdgeDomainSettings({

- Create wildcard DNS records for{" "} + Create a wildcard CNAME record for{" "} *. {initialAutoSubdomainDomain ?? "your automatic subdomain domain"} {" "} - that resolve to every proxy server public IPv4 address. + that points to the edge domain. In production, the edge domain + should resolve to a stable external load balancer with active health + checks.

diff --git a/web/db/schema.ts b/web/db/schema.ts index bac1f16f..5927f39e 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -411,6 +411,10 @@ export const services = pgTable( lastAutomaticPlacementAt: timestamp("last_automatic_placement_at", { withTimezone: true, }), + lastAutomaticRecoveryAttemptAt: timestamp( + "last_automatic_recovery_attempt_at", + { withTimezone: true }, + ), stateful: boolean("stateful").notNull().default(false), lockedServerId: text("locked_server_id").references(() => servers.id, { onDelete: "set null", diff --git a/web/lib/inngest/functions/crons.ts b/web/lib/inngest/functions/crons.ts index f64e12e2..da5ef12c 100644 --- a/web/lib/inngest/functions/crons.ts +++ b/web/lib/inngest/functions/crons.ts @@ -10,6 +10,7 @@ import { checkAndRunScheduledDeployments, cleanupStaleItems, failTimedOutAgentUpgrades, + MAX_AUTOMATIC_RECOVERIES_PER_RUN, rebalanceAutomaticServices, recoverInvalidAutomaticPlacements, } from "@/lib/scheduler"; @@ -22,15 +23,25 @@ export const staleServerCheck = inngest.createFunction( singleton: { mode: "skip" }, }, async ({ step }) => { - await step.run("check-stale-servers", async () => { + const urgentCreated = await step.run("check-stale-servers", async () => { console.log("[cron] running stale server check"); - await checkAndRecoverStaleServers(); - }); - await step.run("recover-invalid-automatic-placements", async () => { - await recoverInvalidAutomaticPlacements(); + return checkAndRecoverStaleServers(); }); + const recoveredCreated = await step.run( + "recover-invalid-automatic-placements", + async () => { + return recoverInvalidAutomaticPlacements( + Math.max(0, MAX_AUTOMATIC_RECOVERIES_PER_RUN - urgentCreated), + ); + }, + ); await step.run("rebalance-automatic-services", async () => { - await rebalanceAutomaticServices(); + await rebalanceAutomaticServices( + Math.max( + 0, + MAX_AUTOMATIC_RECOVERIES_PER_RUN - urgentCreated - recoveredCreated, + ), + ); }); }, ); diff --git a/web/lib/public-api.ts b/web/lib/public-api.ts index 0a47004b..7b021a62 100644 --- a/web/lib/public-api.ts +++ b/web/lib/public-api.ts @@ -693,10 +693,10 @@ export async function patchConfiguration( const source = resolvePersistedSourceFromRows(persisted, repo); if ( input.placement?.mode === "automatic" && - (persisted.stateful || volumes.length > 0) + (persisted.stateful || persisted.serverlessEnabled || volumes.length > 0) ) { domainError( - "Automatic placement is not supported for stateful services or services with volumes", + "Automatic placement is not supported for stateful, serverless, or volume-backed services", "AUTOMATIC_PLACEMENT_UNSUPPORTED", 400, ); @@ -943,7 +943,14 @@ export async function patchConfiguration( .map(({ serverId, count }) => ({ serverId, count })) .toSorted((a, b) => a.serverId.localeCompare(b.serverId)), }, - input.placement, + input.placement.mode === "manual" + ? { + ...input.placement, + placements: input.placement.placements.toSorted((a, b) => + a.serverId.localeCompare(b.serverId), + ), + } + : input.placement, ) ) { set.placementMode = input.placement.mode; diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index 1e9e0f66..9d2aae8e 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -38,6 +38,7 @@ import { const STALE_THRESHOLD_MS = 75 * SECOND_IN_MILLISECONDS; export const AUTOMATIC_PLACEMENT_COOLDOWN_MS = 30 * MINUTE_IN_MILLISECONDS; export const MAX_REBALANCES_PER_RUN = 5; +export const MAX_AUTOMATIC_RECOVERIES_PER_RUN = 5; async function enqueueSystemRollout( serviceId: string, @@ -66,7 +67,10 @@ async function enqueueSystemRollout( } } -export async function rebalanceAutomaticServices(): Promise { +export async function rebalanceAutomaticServices( + maxCreated = MAX_REBALANCES_PER_RUN, +): Promise { + if (maxCreated <= 0) return 0; const now = new Date(); const candidates = await db .select() @@ -75,7 +79,7 @@ export async function rebalanceAutomaticServices(): Promise { .orderBy(services.id); let queuedCount = 0; for (const service of candidates) { - if (queuedCount >= MAX_REBALANCES_PER_RUN) break; + if (queuedCount >= maxCreated) break; if ( service.lastAutomaticPlacementAt && now.getTime() - service.lastAutomaticPlacementAt.getTime() < @@ -169,15 +173,19 @@ export async function rebalanceAutomaticServices(): Promise { active.revisionId, ); if (!result.created) continue; - await enqueueSystemRollout(service.id, result); queuedCount++; + await enqueueSystemRollout(service.id, result); } catch (error) { console.error(`[scheduler] failed to rebalance ${service.name}`, error); } } + return queuedCount; } -export async function recoverInvalidAutomaticPlacements(): Promise { +export async function recoverInvalidAutomaticPlacements( + maxCreated = MAX_AUTOMATIC_RECOVERIES_PER_RUN, +): Promise { + if (maxCreated <= 0) return 0; const activeDeployments = await db .select({ serviceId: deployments.serviceId, @@ -186,8 +194,7 @@ export async function recoverInvalidAutomaticPlacements(): Promise { specification: serviceRevisions.specification, serverStatus: servers.status, serverWireguardIp: servers.wireguardIp, - serverIsProxy: servers.isProxy, - serverOnlineSince: servers.onlineSince, + lastRecoveryAttemptAt: services.lastAutomaticRecoveryAttemptAt, }) .from(deployments) .innerJoin(services, eq(services.id, deployments.serviceId)) @@ -202,7 +209,8 @@ export async function recoverInvalidAutomaticPlacements(): Promise { eq(deployments.trafficState, "active"), isNull(services.deletedAt), ), - ); + ) + .orderBy(deployments.serviceId, deployments.id); const byService = new Map(); for (const deployment of activeDeployments) { @@ -211,65 +219,84 @@ export async function recoverInvalidAutomaticPlacements(): Promise { byService.set(deployment.serviceId, current); } - for (const [serviceId, serviceDeployments] of byService) { - const revisionIds = new Set( - serviceDeployments.map((deployment) => deployment.revisionId), - ); - if (revisionIds.size !== 1) { - console.error( - `[scheduler] skipping automatic recovery for ${serviceId}: multiple active revisions`, + const orderedServices = [...byService.entries()].sort( + ([serviceIdA, deploymentsA], [serviceIdB, deploymentsB]) => { + const attemptedAtA = deploymentsA[0]?.lastRecoveryAttemptAt?.getTime(); + const attemptedAtB = deploymentsB[0]?.lastRecoveryAttemptAt?.getTime(); + if (attemptedAtA === undefined && attemptedAtB !== undefined) return -1; + if (attemptedAtA !== undefined && attemptedAtB === undefined) return 1; + if (attemptedAtA !== attemptedAtB) + return (attemptedAtA ?? 0) - (attemptedAtB ?? 0); + return serviceIdA.localeCompare(serviceIdB); + }, + ); + let createdCount = 0; + for (const [serviceId, serviceDeployments] of orderedServices) { + if (createdCount >= maxCreated) break; + try { + const revisionIds = new Set( + serviceDeployments.map((deployment) => deployment.revisionId), ); - continue; - } - const active = serviceDeployments[0]; - if (!active) continue; - const specification = parseServiceRevisionSpec(active.specification); - if (specification.stateful || specification.placement.mode !== "automatic") - continue; - const hasInvalidPlacement = serviceDeployments.some( - (deployment) => - deployment.serverStatus !== "online" || - !deployment.serverWireguardIp || - !deployment.serverOnlineSince || - deployment.serverOnlineSince.getTime() >= - Date.now() - AUTOMATIC_PLACEMENT_SERVER_STABILIZATION_MS || - (specification.serverless.enabled && !deployment.serverIsProxy), - ); - if (!hasInvalidPlacement) continue; - - const pending = await db - .select({ id: rollouts.id }) - .from(rollouts) - .where( - and( - eq(rollouts.serviceId, serviceId), - inArray(rollouts.status, ["queued", "in_progress"]), - ), + if (revisionIds.size !== 1) { + console.error( + `[scheduler] skipping automatic recovery for ${serviceId}: multiple active revisions`, + ); + continue; + } + const active = serviceDeployments[0]; + if (!active) continue; + const specification = parseServiceRevisionSpec(active.specification); + if ( + specification.stateful || + specification.placement.mode !== "automatic" ) - .limit(1) - .then((rows) => rows[0]); - if (pending) continue; + continue; + const hasInvalidPlacement = serviceDeployments.some( + (deployment) => + deployment.serverStatus !== "online" || !deployment.serverWireguardIp, + ); + if (!hasInvalidPlacement) continue; - try { + const pending = await db + .select({ id: rollouts.id }) + .from(rollouts) + .where( + and( + eq(rollouts.serviceId, serviceId), + inArray(rollouts.status, ["queued", "in_progress"]), + ), + ) + .limit(1) + .then((rows) => rows[0]); + if (pending) continue; + + await db + .update(services) + .set({ lastAutomaticRecoveryAttemptAt: new Date() }) + .where(eq(services.id, serviceId)); await resolveRevisionPlacements(specification); const result = await cloneActiveRevisionAndQueueSystemRollout( serviceId, active.revisionId, ); + if (!result.created) continue; + createdCount++; await enqueueSystemRollout(serviceId, result); } catch (error) { console.error( - `[scheduler] failed level-triggered recovery for ${active.serviceName}`, + `[scheduler] failed level-triggered recovery for ${serviceDeployments[0]?.serviceName ?? serviceId}`, error, ); } } + return createdCount; } async function triggerRecoveryForOfflineServers( offlineServerIds: string[], -): Promise { - if (offlineServerIds.length === 0) return; + maxCreated: number, +): Promise { + if (offlineServerIds.length === 0 || maxCreated <= 0) return 0; const affectedDeployments = await db .select({ @@ -282,6 +309,7 @@ async function triggerRecoveryForOfflineServers( serviceId: services.id, serviceRevisionId: deployments.serviceRevisionId, specification: serviceRevisions.specification, + trafficState: deployments.trafficState, }) .from(deployments) .innerJoin(servers, eq(servers.id, deployments.serverId)) @@ -294,39 +322,65 @@ async function triggerRecoveryForOfflineServers( and( inArray(deployments.serverId, offlineServerIds), inArray(deployments.runtimeDesiredState, ["running", "stopped"]), - eq(deployments.trafficState, "active"), + inArray(deployments.trafficState, ["candidate", "active"]), isNull(services.deletedAt), ), - ); + ) + .orderBy(deployments.serviceId, deployments.id); if (affectedDeployments.length === 0) { console.log( `[scheduler] ${offlineServerIds.length} server(s) went offline; no active replicas need manual recovery`, ); - return; + return 0; } - const automaticallyRecovered = new Set(); - const recoveryAttempted = new Set(); + const automaticActiveByService = new Map< + string, + (typeof affectedDeployments)[number] + >(); + const manualDeploymentIds = new Set(); for (const deployment of affectedDeployments) { - if (recoveryAttempted.has(deployment.serviceId)) continue; - recoveryAttempted.add(deployment.serviceId); try { const specification = parseServiceRevisionSpec(deployment.specification); - if ( - specification.stateful || - specification.placement.mode !== "automatic" - ) + if (specification.placement.mode === "manual") { + manualDeploymentIds.add(deployment.deploymentId); continue; + } + if ( + !specification.stateful && + deployment.trafficState === "active" && + !automaticActiveByService.has(deployment.serviceId) + ) { + automaticActiveByService.set(deployment.serviceId, deployment); + } + } catch (error) { + console.error( + `[scheduler] cannot classify deployment ${deployment.deploymentId} for recovery`, + error, + ); + } + } + + let createdCount = 0; + for (const deployment of automaticActiveByService.values()) { + if (createdCount >= maxCreated) break; + try { + const specification = parseServiceRevisionSpec(deployment.specification); + await db + .update(services) + .set({ lastAutomaticRecoveryAttemptAt: new Date() }) + .where(eq(services.id, deployment.serviceId)); await resolveRevisionPlacements(specification); const queued = await cloneActiveRevisionAndQueueSystemRollout( deployment.serviceId, deployment.serviceRevisionId, ); + if (!queued.created) continue; + createdCount++; await enqueueSystemRollout(deployment.serviceId, queued); - automaticallyRecovered.add(deployment.serviceId); } catch (error) { console.error( - `[scheduler] automatic recovery failed for ${deployment.serviceName}; falling back to manual alert`, + `[scheduler] automatic recovery failed for ${deployment.serviceName}; periodic recovery will retry`, error, ); } @@ -343,7 +397,7 @@ async function triggerRecoveryForOfflineServers( >(); for (const deployment of affectedDeployments) { - if (automaticallyRecovered.has(deployment.serviceId)) continue; + if (!manualDeploymentIds.has(deployment.deploymentId)) continue; const current = affectedByServer.get(deployment.serverId) ?? { serverName: deployment.serverName, serverIp: @@ -373,11 +427,12 @@ async function triggerRecoveryForOfflineServers( ); }); } + return createdCount; } export async function checkAndRecoverStaleServers( excludeServerId?: string, -): Promise { +): Promise { const staleThreshold = subtractMilliseconds(new Date(), STALE_THRESHOLD_MS); const conditions = [ @@ -400,7 +455,7 @@ export async function checkAndRecoverStaleServers( wireguardIp: servers.wireguardIp, }); - if (markedOffline.length === 0) return; + if (markedOffline.length === 0) return 0; const offlineIds = markedOffline.map((s) => s.id); console.log( @@ -419,7 +474,10 @@ export async function checkAndRecoverStaleServers( }); } - await triggerRecoveryForOfflineServers(offlineIds); + return triggerRecoveryForOfflineServers( + offlineIds, + MAX_AUTOMATIC_RECOVERIES_PER_RUN, + ); } export async function checkAndRunScheduledDeployments(): Promise { diff --git a/web/lib/service-revision-changes.ts b/web/lib/service-revision-changes.ts index 8ab83abd..21cd9000 100644 --- a/web/lib/service-revision-changes.ts +++ b/web/lib/service-revision-changes.ts @@ -105,6 +105,11 @@ const serviceRevisionSpecSchema = z code: "custom", message: "Stateful services cannot use automatic placement", }); + if (spec.serverless.enabled && spec.placement.mode === "automatic") + context.addIssue({ + code: "custom", + message: "Serverless services cannot use automatic placement", + }); }); export type ServiceRevisionChange = { diff --git a/web/lib/service-revision-spec.ts b/web/lib/service-revision-spec.ts index f400f348..b2307eb6 100644 --- a/web/lib/service-revision-spec.ts +++ b/web/lib/service-revision-spec.ts @@ -161,6 +161,12 @@ function validateServiceRevisionSpec( if (specification.stateful && specification.placement.mode === "automatic") { throw new Error("Stateful services cannot use automatic placement"); } + if ( + specification.serverless.enabled && + specification.placement.mode === "automatic" + ) { + throw new Error("Serverless services cannot use automatic placement"); + } if (specification.stateful && totalReplicas !== 1) { throw new Error("Stateful services can only have exactly 1 replica"); } diff --git a/web/tests/service-revision-changes.test.ts b/web/tests/service-revision-changes.test.ts index 1b068782..0729d1ce 100644 --- a/web/tests/service-revision-changes.test.ts +++ b/web/tests/service-revision-changes.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { diffServiceRevisionSpecs } from "@/lib/service-revision-changes"; +import { + diffServiceRevisionSpecs, + parseServiceRevisionSpec, +} from "@/lib/service-revision-changes"; import type { ServiceRevisionSpec } from "@/lib/service-revision-spec"; function spec(): ServiceRevisionSpec { @@ -156,6 +159,17 @@ describe("diffServiceRevisionSpecs", () => { ]); }); + it("rejects persisted automatic serverless revisions", () => { + const automatic = spec(); + automatic.placement = { mode: "automatic", replicas: 1 }; + automatic.placements = []; + automatic.serverless.enabled = true; + + expect(() => parseServiceRevisionSpec(automatic)).toThrow( + "Serverless services cannot use automatic placement", + ); + }); + it("never exposes secret ciphertext while detecting additions, updates, and removals", () => { const previous = spec(); previous.secrets.push({ diff --git a/web/tests/service-revision-spec.test.ts b/web/tests/service-revision-spec.test.ts index 70d9aaa7..1bf4831b 100644 --- a/web/tests/service-revision-spec.test.ts +++ b/web/tests/service-revision-spec.test.ts @@ -152,7 +152,7 @@ describe("service revision specification", () => { }); }); - it("rejects automatic placement for stateful and volume-backed services", () => { + it("rejects automatic placement for stateful, volume-backed, and serverless services", () => { const stateful = draft({ volumes: [] }); stateful.service.stateful = true; stateful.service.placementMode = "automatic"; @@ -169,6 +169,15 @@ describe("service revision specification", () => { expect(() => buildServiceRevisionSpec(volumeBacked)).toThrow( "Services with volumes cannot use automatic placement", ); + + const serverless = draft({ volumes: [] }); + serverless.service.serverlessEnabled = true; + serverless.service.placementMode = "automatic"; + serverless.service.replicas = 1; + + expect(() => buildServiceRevisionSpec(serverless)).toThrow( + "Serverless services cannot use automatic placement", + ); }); it("rejects serverless revisions without a public HTTP port and domain", () => { From 15fc8d72fd118890476f557b0cd3db59bc85d793 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:05:26 +1000 Subject: [PATCH 03/43] Complete automatic placement contract switch --- cli/internal/cli/app.go | 23 +++++++----- cli/internal/cli/app_test.go | 40 ++++++++++++++++++--- cli/internal/manifest/manifest.go | 3 ++ cli/internal/manifest/manifest_test.go | 13 +++---- docs/api/public-api.mdx | 2 +- web/actions/projects.ts | 10 ++---- web/lib/public-api.ts | 39 -------------------- web/lib/scheduler.ts | 50 ++++++++++++++++++++++++-- web/tests/public-api-source.test.ts | 6 ++++ 9 files changed, 115 insertions(+), 71 deletions(-) diff --git a/cli/internal/cli/app.go b/cli/internal/cli/app.go index cefd7fc7..d4f2b78b 100644 --- a/cli/internal/cli/app.go +++ b/cli/internal/cli/app.go @@ -300,6 +300,8 @@ service: image: nginx:1.27 hostname: null replicas: 1 + placement: + mode: automatic healthCheck: null startCommand: null ports: @@ -479,7 +481,9 @@ func (a *App) linkCommand() *cobra.Command { if cfg.Current.Placement != nil { if cfg.Current.Placement.Mode == "automatic" { placement = &manifest.Placement{Mode: "automatic"} - } else if len(cfg.Current.Placements) > 0 { + } else if len(cfg.Current.Placements) == 0 { + return errors.New("configure at least one server placement in the web UI before linking this service") + } else { placement = &manifest.Placement{Mode: "manual"} placement.Servers = make([]manifest.PlacementServer, len(cfg.Current.Placements)) for i, p := range cfg.Current.Placements { @@ -526,14 +530,15 @@ func (a *App) applyCommand() *cobra.Command { if !loaded.Manifest.Linked() { return errors.New("service is not linked: run `tc link`") } - body := map[string]any{"source": sourcePatch(loaded.Manifest.Service.Source), "hostname": loaded.Manifest.Service.Hostname, "ports": loaded.Manifest.Service.Ports, "replicas": loaded.Manifest.Service.Replicas, "healthCheck": loaded.Manifest.Service.HealthCheck, "startCommand": loaded.Manifest.Service.StartCommand} - if placement := loaded.Manifest.Service.Placement; placement != nil { - delete(body, "replicas") - if placement.Mode == "automatic" { - body["placement"] = map[string]any{"mode": "automatic", "replicas": loaded.Manifest.Service.Replicas} - } else { - body["placement"] = map[string]any{"mode": "manual", "placements": placement.Servers} - } + placement := loaded.Manifest.Service.Placement + if placement == nil { + return errors.New("service.placement is required") + } + body := map[string]any{"source": sourcePatch(loaded.Manifest.Service.Source), "hostname": loaded.Manifest.Service.Hostname, "ports": loaded.Manifest.Service.Ports, "healthCheck": loaded.Manifest.Service.HealthCheck, "startCommand": loaded.Manifest.Service.StartCommand} + if placement.Mode == "automatic" { + body["placement"] = map[string]any{"mode": "automatic", "replicas": loaded.Manifest.Service.Replicas} + } else { + body["placement"] = map[string]any{"mode": "manual", "placements": placement.Servers} } if loaded.Manifest.Service.Resources != nil { body["resources"] = loaded.Manifest.Service.Resources diff --git a/cli/internal/cli/app_test.go b/cli/internal/cli/app_test.go index 2f135542..ba43db82 100644 --- a/cli/internal/cli/app_test.go +++ b/cli/internal/cli/app_test.go @@ -26,6 +26,7 @@ service: name: web source: {type: image, image: nginx:1.27} replicas: 2 + placement: {mode: automatic} hostname: null healthCheck: null startCommand: null @@ -145,6 +146,32 @@ func TestLinkByIDsFetchesConfigurationAndSupportsPublicGitHub(t *testing.T) { } } +func TestLinkRejectsManualServiceWithoutPlacements(t *testing.T) { + configHome(t) + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/projects": + w.Write([]byte(`{"projects":[{"id":"p","name":"App","slug":"app"}]}`)) + case "/api/v1/projects/p/environments": + w.Write([]byte(`{"environments":[{"id":"e","name":"prod"}]}`)) + case "/api/v1/projects/p/environments/e/services": + w.Write([]byte(`{"services":[{"id":"s","name":"web","source":{"type":"image","image":"nginx"}}]}`)) + case "/api/v1/projects/p/environments/e/services/s/configuration": + w.Write([]byte(`{"current":{"replicas":0,"placement":{"mode":"manual"},"placements":[],"hostname":null,"ports":[],"healthCheck":null,"startCommand":null},"management":{"patchable":true,"blockers":[]}}`)) + default: + t.Errorf("path=%s", r.URL.Path) + } + })) + defer s.Close() + writeConfig(t, s.URL) + d := t.TempDir() + app, _ := testApp(t, d, s.Client()) + err := execute(app, "link", "--project", "p", "--environment", "e", "--service", "s") + if err == nil || !strings.Contains(err.Error(), "configure at least one server placement") { + t.Fatalf("error = %v", err) + } +} + func TestApplyExactNestedPatchForSources(t *testing.T) { for _, tc := range []struct{ name, source, sourceType string }{ {"image", "{type: image, image: nginx:1.27}", "image"}, @@ -172,7 +199,8 @@ func TestApplyExactNestedPatchForSources(t *testing.T) { t.Fatalf("%s %s", method, path) } source := body["source"].(map[string]any) - if source["type"] != tc.sourceType || body["replicas"] != float64(2) || len(body) != 6 { + placement := body["placement"].(map[string]any) + if source["type"] != tc.sourceType || placement["mode"] != "automatic" || placement["replicas"] != float64(2) || len(body) != 6 { t.Fatalf("body=%#v", body) } if tc.name == "github_clear_root" { @@ -190,13 +218,17 @@ func TestApplyPlacementPayloads(t *testing.T) { name, yaml string want map[string]any }{ - {"automatic", " placement: {mode: automatic}\n", map[string]any{"mode": "automatic", "replicas": float64(2)}}, + {"automatic", "", map[string]any{"mode": "automatic", "replicas": float64(2)}}, {"manual", " placement:\n mode: manual\n servers:\n - {serverId: server-a, count: 2}\n", map[string]any{"mode": "manual", "placements": []any{map[string]any{"serverId": "server-a", "count": float64(2)}}}}, } { t.Run(tc.name, func(t *testing.T) { configHome(t) d := t.TempDir() - writeManifest(t, d, imageManifest+tc.yaml) + manifestYAML := imageManifest + if tc.yaml != "" { + manifestYAML = strings.Replace(manifestYAML, " placement: {mode: automatic}\n", tc.yaml, 1) + } + writeManifest(t, d, manifestYAML) var body map[string]any s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { json.NewDecoder(r.Body).Decode(&body) @@ -209,7 +241,7 @@ func TestApplyPlacementPayloads(t *testing.T) { t.Fatal(err) } if _, exists := body["replicas"]; exists { - t.Fatalf("legacy replicas present: %#v", body) + t.Fatalf("top-level replicas present: %#v", body) } if !reflect.DeepEqual(body["placement"], tc.want) { t.Fatalf("placement=%#v want=%#v", body["placement"], tc.want) diff --git a/cli/internal/manifest/manifest.go b/cli/internal/manifest/manifest.go index cb1ba4e0..76136a77 100644 --- a/cli/internal/manifest/manifest.go +++ b/cli/internal/manifest/manifest.go @@ -228,6 +228,9 @@ func Validate(m Manifest) error { if m.Service.Replicas < 1 || m.Service.Replicas > 10 { return errors.New("service.replicas must be between 1 and 10") } + if m.Service.Placement == nil { + return errors.New("service.placement is required") + } if p := m.Service.Placement; p != nil { switch p.Mode { case "automatic": diff --git a/cli/internal/manifest/manifest_test.go b/cli/internal/manifest/manifest_test.go index 5b7318e4..83e3a025 100644 --- a/cli/internal/manifest/manifest_test.go +++ b/cli/internal/manifest/manifest_test.go @@ -6,7 +6,7 @@ import ( ) func base() Manifest { - return Manifest{APIVersion: "v1", Project: Project{ID: "p", Slug: "app"}, Environment: Environment{ID: "e", Name: "prod"}, Service: Service{ID: "s", Name: "web", Source: Source{Type: "image", Image: "nginx"}, Replicas: 1}} + return Manifest{APIVersion: "v1", Project: Project{ID: "p", Slug: "app"}, Environment: Environment{ID: "e", Name: "prod"}, Service: Service{ID: "s", Name: "web", Source: Source{Type: "image", Image: "nginx"}, Replicas: 1, Placement: &Placement{Mode: "automatic"}}} } func TestDefaultsAndRoundTrip(t *testing.T) { m := base() @@ -63,8 +63,8 @@ func TestPlacementRoundTripAndValidation(t *testing.T) { } } -func TestPlacementOmittedIsBackwardCompatible(t *testing.T) { - m, err := Parse([]byte(`apiVersion: v1 +func TestPlacementIsRequired(t *testing.T) { + _, err := Parse([]byte(`apiVersion: v1 project: {slug: app} environment: {name: prod} service: @@ -72,11 +72,8 @@ service: source: {type: image, image: nginx} replicas: 2 `)) - if err != nil { - t.Fatal(err) - } - if m.Service.Placement != nil || m.Service.Replicas != 2 { - t.Fatalf("service=%#v", m.Service) + if err == nil || !strings.Contains(err.Error(), "service.placement is required") { + t.Fatalf("error = %v", err) } } func TestGitHubCanonical(t *testing.T) { diff --git a/docs/api/public-api.mdx b/docs/api/public-api.mdx index 70e9ce53..5fb9c8ae 100644 --- a/docs/api/public-api.mdx +++ b/docs/api/public-api.mdx @@ -209,7 +209,7 @@ Use manual placement to choose exact servers: } ``` -Manual placement requires online servers with WireGuard configured. Serverless services require proxy servers. Automatic placement is not available for stateful, serverless, or volume-backed services. The legacy top-level `replicas` field remains supported when placement is unchanged. It must match the existing manual placement. +Manual placement requires online servers with WireGuard configured. Serverless services require proxy servers. Automatic placement is not available for stateful, serverless, or volume-backed services. Submit replica changes through `placement`; the API rejects a top-level `replicas` field. The API only manages stateless services with HTTP ports. Existing volumes, stateful mode, TCP or UDP ports, TLS passthrough, or invalid resource limits return a conflict with an actionable code. diff --git a/web/actions/projects.ts b/web/actions/projects.ts index 45e79701..777d98b8 100644 --- a/web/actions/projects.ts +++ b/web/actions/projects.ts @@ -989,7 +989,6 @@ export type ServiceConfigUpdate = { source?: { type: "image"; image: string }; healthCheck?: ServiceHealthCheckConfig | null; ports?: { add?: PortConfig[]; remove?: string[] }; - replicas?: { serverId: string; count: number }[]; placement?: | { mode: "automatic"; replicas: number } | { @@ -1154,13 +1153,8 @@ export async function updateServiceConfig( } } - if (config.placement || config.replicas) { - const placement = placementInputSchema.parse( - config.placement ?? { - mode: "manual", - placements: config.replicas?.filter((replica) => replica.count > 0), - }, - ); + if (config.placement) { + const placement = placementInputSchema.parse(config.placement); await db.transaction(async (tx) => { await tx.execute( sql`SELECT pg_advisory_xact_lock(hashtext(${serviceId}))`, diff --git a/web/lib/public-api.ts b/web/lib/public-api.ts index 7b021a62..863688dc 100644 --- a/web/lib/public-api.ts +++ b/web/lib/public-api.ts @@ -594,7 +594,6 @@ export const configurationPatchSchema = z.strictObject({ source: publicSourceSchema.optional(), hostname: hostnameSchema.nullable().optional(), ports: z.array(portSchema).max(100).optional(), - replicas: z.number().int().min(1).max(10).optional(), placement: placementSchema.optional(), healthCheck: healthCheckSchema.nullable().optional(), startCommand: z.string().trim().min(1).max(4096).nullable().optional(), @@ -686,10 +685,6 @@ export async function patchConfiguration( .limit(1) .then((rows) => rows[0]), ]); - const replicaCount = - persisted.placementMode === "automatic" - ? persisted.replicas - : placements.reduce((sum, placement) => sum + placement.count, 0); const source = resolvePersistedSourceFromRows(persisted, repo); if ( input.placement?.mode === "automatic" && @@ -734,34 +729,6 @@ export async function patchConfiguration( ); } } - if ( - input.replicas !== undefined && - input.placement && - input.replicas !== - (input.placement.mode === "automatic" - ? input.placement.replicas - : input.placement.placements.reduce( - (sum, item) => sum + item.count, - 0, - )) - ) { - domainError( - "replicas and placement describe different desired replica counts", - "REPLICA_PLACEMENT_MISMATCH", - 400, - ); - } - if ( - input.replicas !== undefined && - !input.placement && - persisted.placementMode === "manual" && - input.replicas !== replicaCount - ) { - domainError( - `replicas must match the current manual placement of ${replicaCount}`, - "REPLICA_PLACEMENT_MISMATCH", - ); - } if (input.placement?.mode === "manual") { const ids = input.placement.placements.map((item) => item.serverId); const selected = await tx @@ -967,12 +934,6 @@ export async function patchConfiguration( })), ); } - } else if ( - input.replicas !== undefined && - persisted.placementMode === "automatic" && - changed("placement.replicas", persisted.replicas, input.replicas) - ) { - set.replicas = input.replicas; } if (input.source?.type === "github") { const effectiveBranch = diff --git a/web/lib/scheduler.ts b/web/lib/scheduler.ts index 9d2aae8e..7a4fe970 100644 --- a/web/lib/scheduler.ts +++ b/web/lib/scheduler.ts @@ -73,9 +73,29 @@ export async function rebalanceAutomaticServices( if (maxCreated <= 0) return 0; const now = new Date(); const candidates = await db - .select() + .select({ + id: services.id, + name: services.name, + lastAutomaticPlacementAt: services.lastAutomaticPlacementAt, + }) .from(services) - .where(and(isNull(services.deletedAt))) + .innerJoin(deployments, eq(deployments.serviceId, services.id)) + .innerJoin( + serviceRevisions, + eq(serviceRevisions.id, deployments.serviceRevisionId), + ) + .where( + and( + isNull(services.deletedAt), + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + eq( + sql`${serviceRevisions.specification} -> 'placement' ->> 'mode'`, + "automatic", + ), + ), + ) + .groupBy(services.id, services.name, services.lastAutomaticPlacementAt) .orderBy(services.id); let queuedCount = 0; for (const service of candidates) { @@ -186,6 +206,28 @@ export async function recoverInvalidAutomaticPlacements( maxCreated = MAX_AUTOMATIC_RECOVERIES_PER_RUN, ): Promise { if (maxCreated <= 0) return 0; + const candidateServices = await db + .select({ serviceId: deployments.serviceId }) + .from(deployments) + .innerJoin(services, eq(services.id, deployments.serviceId)) + .innerJoin( + serviceRevisions, + eq(serviceRevisions.id, deployments.serviceRevisionId), + ) + .where( + and( + inArray(deployments.runtimeDesiredState, ["running", "stopped"]), + eq(deployments.trafficState, "active"), + isNull(services.deletedAt), + eq( + sql`${serviceRevisions.specification} -> 'placement' ->> 'mode'`, + "automatic", + ), + ), + ) + .groupBy(deployments.serviceId); + if (candidateServices.length === 0) return 0; + const activeDeployments = await db .select({ serviceId: deployments.serviceId, @@ -208,6 +250,10 @@ export async function recoverInvalidAutomaticPlacements( inArray(deployments.runtimeDesiredState, ["running", "stopped"]), eq(deployments.trafficState, "active"), isNull(services.deletedAt), + inArray( + deployments.serviceId, + candidateServices.map((service) => service.serviceId), + ), ), ) .orderBy(deployments.serviceId, deployments.id); diff --git a/web/tests/public-api-source.test.ts b/web/tests/public-api-source.test.ts index 71189f94..030625a6 100644 --- a/web/tests/public-api-source.test.ts +++ b/web/tests/public-api-source.test.ts @@ -115,6 +115,12 @@ describe("public API GitHub sources", () => { }); describe("public API placement schema", () => { + it("rejects the removed standalone replicas field", () => { + expect(configurationPatchSchema.safeParse({ replicas: 3 }).success).toBe( + false, + ); + }); + it.each([ { mode: "automatic", replicas: 3 }, { mode: "manual", placements: [{ serverId: "server-1", count: 2 }] }, From b8fd1cedd3f82d174b35e4c96119aeba922511e2 Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 23 Jul 2026 07:32:53 +0000 Subject: [PATCH 04/43] Combine API keys into security settings Amp-Thread-ID: https://ampcode.com/threads/T-019f8ddb-567f-7098-b526-777a2b97c8ec Co-authored-by: Arjun Komath --- web/components/settings/global-settings.tsx | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/web/components/settings/global-settings.tsx b/web/components/settings/global-settings.tsx index 63de5009..102c6876 100644 --- a/web/components/settings/global-settings.tsx +++ b/web/components/settings/global-settings.tsx @@ -108,6 +108,7 @@ export function GlobalSettings({ const [tab, setTab] = useQueryState("tab", { defaultValue: "infrastructure", }); + const activeTab = tab === "api-keys" ? "security" : tab; const previousTabRef = useRef(null); const [buildServerIds, setBuildServerIds] = useState>( new Set(initialSettings.buildServerIds), @@ -125,6 +126,12 @@ export function GlobalSettings({ const [controlPlaneUpgradeDialogOpen, setControlPlaneUpgradeDialogOpen] = useState(false); + useEffect(() => { + if (tab === "api-keys") { + void setTab("security", { history: "replace" }); + } + }, [tab, setTab]); + useEffect(() => { const openedAbout = tab === "update" && previousTabRef.current !== "update"; previousTabRef.current = tab; @@ -258,7 +265,7 @@ export function GlobalSettings({ const upgradeRunning = upgradeState?.status === "running"; return ( - setTab(value)}> + setTab(value)}> Infrastructure @@ -269,9 +276,6 @@ export function GlobalSettings({ Security - - API Keys - {membersData && ( Members @@ -451,9 +455,6 @@ export function GlobalSettings({ - - - From c4b816a213465b1326c1a5ea43fad060b10a3f78 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:05:03 +1000 Subject: [PATCH 05/43] Enable deploy for automatic placement --- .../service/details/deployment-progress.tsx | 11 +++++++---- .../service/details/pending-changes-banner.tsx | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/web/components/service/details/deployment-progress.tsx b/web/components/service/details/deployment-progress.tsx index 87fb1742..56543f2a 100644 --- a/web/components/service/details/deployment-progress.tsx +++ b/web/components/service/details/deployment-progress.tsx @@ -150,10 +150,13 @@ export function getBarState( } } - const totalReplicas = service.configuredReplicas.reduce( - (sum, r) => sum + r.count, - 0, - ); + const totalReplicas = + service.placementMode === "automatic" + ? service.replicas + : service.configuredReplicas.reduce( + (sum, replica) => sum + replica.count, + 0, + ); const hasNoDeployments = service.deployments.length === 0; const hasChanges = changes.length > 0; diff --git a/web/components/service/details/pending-changes-banner.tsx b/web/components/service/details/pending-changes-banner.tsx index 46cd2c03..6eac261c 100644 --- a/web/components/service/details/pending-changes-banner.tsx +++ b/web/components/service/details/pending-changes-banner.tsx @@ -35,10 +35,13 @@ export const PendingChangesBanner = memo(function PendingChangesBanner({ const { mutate } = useSWRConfig(); const [isDeploying, setIsDeploying] = useState(false); - const totalReplicas = service.configuredReplicas.reduce( - (sum, r) => sum + r.count, - 0, - ); + const totalReplicas = + service.placementMode === "automatic" + ? service.replicas + : service.configuredReplicas.reduce( + (sum, replica) => sum + replica.count, + 0, + ); const hasNoDeployments = service.deployments.length === 0; const shouldBuild = service.sourceType === "github" && From 12d999685bcd509d4229009da35ce9629b0346dd Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 23 Jul 2026 08:06:23 +0000 Subject: [PATCH 06/43] Remove legacy API keys tab alias Amp-Thread-ID: https://ampcode.com/threads/T-019f8ddb-567f-7098-b526-777a2b97c8ec Co-authored-by: Arjun Komath --- web/components/settings/global-settings.tsx | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/web/components/settings/global-settings.tsx b/web/components/settings/global-settings.tsx index 102c6876..e131fd23 100644 --- a/web/components/settings/global-settings.tsx +++ b/web/components/settings/global-settings.tsx @@ -108,7 +108,6 @@ export function GlobalSettings({ const [tab, setTab] = useQueryState("tab", { defaultValue: "infrastructure", }); - const activeTab = tab === "api-keys" ? "security" : tab; const previousTabRef = useRef(null); const [buildServerIds, setBuildServerIds] = useState>( new Set(initialSettings.buildServerIds), @@ -126,12 +125,6 @@ export function GlobalSettings({ const [controlPlaneUpgradeDialogOpen, setControlPlaneUpgradeDialogOpen] = useState(false); - useEffect(() => { - if (tab === "api-keys") { - void setTab("security", { history: "replace" }); - } - }, [tab, setTab]); - useEffect(() => { const openedAbout = tab === "update" && previousTabRef.current !== "update"; previousTabRef.current = tab; @@ -265,7 +258,7 @@ export function GlobalSettings({ const upgradeRunning = upgradeState?.status === "running"; return ( - setTab(value)}> + setTab(value)}> Infrastructure From c7757108917f89ca8b1dcc8e03c712f6d6e42d28 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:24:19 +1000 Subject: [PATCH 07/43] Allow cross-platform build fallback --- web/lib/build-assignment.ts | 6 ++++-- web/tests/build-assignment.test.ts | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/web/lib/build-assignment.ts b/web/lib/build-assignment.ts index 7239d1dc..1aea26aa 100644 --- a/web/lib/build-assignment.ts +++ b/web/lib/build-assignment.ts @@ -80,10 +80,12 @@ export async function selectBuildServerForRevision( const matchingServers = onlineServers.filter( (server) => server.meta?.arch === arch, ); - if (matchingServers.length === 0) { + const buildServers = + matchingServers.length > 0 ? matchingServers : onlineServers; + if (buildServers.length === 0) { throw new Error(`No online servers available for platform ${platform}`); } - return matchingServers[Math.floor(Math.random() * matchingServers.length)].id; + return buildServers[Math.floor(Math.random() * buildServers.length)].id; } export async function getTargetPlatformsForRevision( diff --git a/web/tests/build-assignment.test.ts b/web/tests/build-assignment.test.ts index fbe1d133..7e16602a 100644 --- a/web/tests/build-assignment.test.ts +++ b/web/tests/build-assignment.test.ts @@ -155,6 +155,14 @@ describe("revision-backed build assignment", () => { ).resolves.toBe("server-arm"); }); + it("falls back to cross-platform builds when no native builder is online", async () => { + mocks.getSetting.mockResolvedValue(null); + mocks.queryResults.push([{ id: "server-arm", meta: { arch: "arm64" } }]); + await expect( + selectBuildServerForRevision(specification(), "linux/amd64"), + ).resolves.toBe("server-arm"); + }); + it("limits stateless build assignment to configured build servers", async () => { mocks.getSetting.mockResolvedValue(["server-arm"]); mocks.queryResults.push([{ id: "server-arm", meta: { arch: "arm64" } }]); From c4a4e268cf44c1a8c9102f64b57bdff4d28782cf Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:46:27 +1000 Subject: [PATCH 08/43] Constrain placement mode picker width --- web/components/service/details/replicas-section.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/components/service/details/replicas-section.tsx b/web/components/service/details/replicas-section.tsx index 2a1df621..b0f071d8 100644 --- a/web/components/service/details/replicas-section.tsx +++ b/web/components/service/details/replicas-section.tsx @@ -383,7 +383,7 @@ export const ReplicasSection = memo(function ReplicasSection({ >
- + {!service.serverlessEnabled && ( Automatic From 57c56e5b5fa3a0a425358fee8cff35776b19a132 Mon Sep 17 00:00:00 2001 From: Techulus Agent <291950465+techulus-agent@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:49:21 +1000 Subject: [PATCH 09/43] Simplify automatic placement controls --- web/components/service/details/replicas-section.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/components/service/details/replicas-section.tsx b/web/components/service/details/replicas-section.tsx index b0f071d8..280ddee3 100644 --- a/web/components/service/details/replicas-section.tsx +++ b/web/components/service/details/replicas-section.tsx @@ -396,7 +396,7 @@ export const ReplicasSection = memo(function ReplicasSection({ {placementMode === "automatic" ? ( -
+