From 53fb08b9bfc5d871a92d25a9b2777e9bc3f260f6 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Tue, 6 Oct 2026 00:02:55 +0200 Subject: [PATCH] chore(infra): route electric through ingest's ALB instead of its own Each region ran a dedicated ALB for electric that served a few thousand requests a day, costing about $16-20/mo per region for its hourly charge alone. Ingest now composes its ALB and listener explicitly; electric attaches a host rule (priority 10) and its own SNI certificate to that listener, while ingest keeps the catch-all (priority 50000). Both services move to new target groups, since an ALB target group can only belong to one load balancer: the first deploy rolls ingest and gives electric its usual ~60s gap. Electric's ALB security group goes away; the shared ALB admits only Cloudflare, which is how electric-sync reaches it. --- alchemy.run.ts | 8 ++-- apps/electric/alchemy.run.ts | 80 +++++++++++++++++------------------- apps/ingest/alchemy.run.ts | 42 ++++++++++++++++--- 3 files changed, 80 insertions(+), 50 deletions(-) diff --git a/alchemy.run.ts b/alchemy.run.ts index bcc9f941d..8d498174d 100644 --- a/alchemy.run.ts +++ b/alchemy.run.ts @@ -247,7 +247,7 @@ export default Alchemy.Stack( yield* serveWorker("api", api) // Not wired into electric-sync: it reads `ELECTRIC_URL` from the secret store, so - // cutover is separate. Electric runs in ingest's VPC, hence `ingest &&`. + // cutover is separate. Electric runs in ingest's VPC behind its ALB (a host rule), hence `ingest &&`. // Id must not be `"electric"` (the ECS service's id): alchemy keys state by id alone. const electricDbRole = db && profile.deploys.electric @@ -259,13 +259,15 @@ export default Alchemy.Stack( }) : undefined const electric = - ingest && electricDbRole + ingest && electricDbRole && domains.electric ? yield* createMapleElectric({ stage, region, - domains, profile, network: ingest.network, + listener: ingest.listener, + albSecurityGroupId: ingest.albSecurityGroupId, + hostname: domains.electric, dbRole: electricDbRole, }) : undefined diff --git a/apps/electric/alchemy.run.ts b/apps/electric/alchemy.run.ts index 043012e7e..94ddd39fd 100644 --- a/apps/electric/alchemy.run.ts +++ b/apps/electric/alchemy.run.ts @@ -14,27 +14,32 @@ const ELECTRIC_PORT = 3000 /** Absolute: alchemy has changed how a relative `dockerfile` resolves between releases. */ const DOCKERFILE = resolve("apps/electric/Dockerfile") -export interface CreateMapleElectricOptions extends Pick< - MapleStackContext, - "stage" | "region" | "domains" | "profile" -> { +export interface CreateMapleElectricOptions extends Pick { /** The ingest VPC: a second `AWS.EC2.Network` in one stack fights over the internet gateway. */ network: Pick + /** Ingest's ALB listener, shared: a dedicated ALB costs more than this service's traffic. */ + listener: AWS.ELBv2.Listener + /** The shared ALB's group, the only source admitted to ELECTRIC_PORT. */ + albSecurityGroupId: AWS.EC2.SecurityGroup["groupId"] + /** Routed by host on the shared listener, so required (prd is the only stage that deploys this). */ + hostname: string /** The replication role on the instance's branch (`withReplication`), minted by the root. */ dbRole: Planetscale.PostgresRole } /** * Self-hosted ElectricSQL on ECS Fargate, the upstream behind `apps/electric-sync`. - * Shares ingest's VPC but has its own cluster, ALB, security groups and certificate. + * Shares ingest's VPC and ALB (a host rule plus its own SNI certificate); own cluster and task group. * Runbook: `docs/electric-sync.md`. */ export const createMapleElectric = ({ stage, region, - domains, profile, network, + listener, + albSecurityGroupId, + hostname, dbRole, }: CreateMapleElectricOptions) => Effect.gen(function* () { @@ -42,27 +47,9 @@ export const createMapleElectric = ({ const name = (base: string) => resolveAwsResourceName(base, stage, region) const tags = { Service: "maple-electric", Region: region } - // Alchemy keys state by logical id: renaming these ids replaces live groups, and a - // new group must also get a new `groupName` or it collides with the old one. - // `securityGroups` apply to both ALB and tasks, so only the ALB's group may reach - // ELECTRIC_PORT; otherwise a task's public IP serves plaintext around the cert. - const listenerPort = domains.electric ? 443 : 80 - const albSecurityGroup = yield* AWS.EC2.SecurityGroup("electric-lb-sg", { - vpcId: network.vpcId, - groupName: name("electric-lb"), - description: `Maple ElectricSQL - public ${listenerPort === 443 ? "HTTPS" : "HTTP"} to the load balancer`, - ingress: [ - { - ipProtocol: "tcp", - fromPort: listenerPort, - toPort: listenerPort, - // Not narrowed to Cloudflare ranges (they rotate); ELECTRIC_SECRET authorizes. - cidrIpv4: "0.0.0.0/0", - description: "Shape requests from the electric-sync Worker", - }, - ], - }) - + // Only the shared ALB may reach ELECTRIC_PORT; otherwise a task's public IP serves + // plaintext around the cert. The ALB itself admits only Cloudflare (ingest's group), + // which is fine: electric-sync reaches this through the proxied hostname. const taskSecurityGroup = yield* AWS.EC2.SecurityGroup("electric-task-sg", { vpcId: network.vpcId, groupName: name("electric-task"), @@ -72,7 +59,7 @@ export const createMapleElectric = ({ ipProtocol: "tcp", fromPort: ELECTRIC_PORT, toPort: ELECTRIC_PORT, - referencedGroupId: albSecurityGroup.groupId, + referencedGroupId: albSecurityGroupId, description: "ALB to task", }, ], @@ -91,12 +78,19 @@ export const createMapleElectric = ({ // Shared with the electric-sync Worker; rotate by redeploying this first, then the Worker. const apiSecret = yield* secret("api-secret", yield* requiredPlain("ELECTRIC_SECRET")) - const issuedCertificateArn = yield* issueRegionalCertificate({ + const certificateArn = yield* issueRegionalCertificate({ id: "electric-cert", - hostname: domains.electric, + hostname, region: resolveAwsRegion(region), tags, }) + // SNI: the listener's default certificate is ingest's. + if (certificateArn) { + yield* AWS.ELBv2.ListenerCertificate("electric-listener-cert", { + listenerArn: listener, + certificateArn, + }) + } const baseEnv = { ELECTRIC_PORT: String(ELECTRIC_PORT), @@ -133,15 +127,19 @@ export const createMapleElectric = ({ vpcId: network.vpcId, subnets: network.publicSubnetIds, - securityGroups: [albSecurityGroup.groupId, taskSecurityGroup.groupId], + securityGroups: [taskSecurityGroup.groupId], assignPublicIp: true, // Public: the caller is a Worker with no route into the VPC; ELECTRIC_SECRET guards it. - // `port` is the container port; the listener goes to 443 once `certificateArn` is set. - public: true, + // The explicit `forward` names a fresh target group: an ALB target group belongs to + // one load balancer, so the one from electric's former ALB can't move here. port: ELECTRIC_PORT, + loadBalancer: { + listener, + // Ahead of ingest's catch-all (priority 50000). + rules: [{ host: hostname, forward: `${ELECTRIC_PORT}/http`, priority: 10 }], + }, healthCheckPath: "/v1/health", - ...(issuedCertificateArn ? { certificateArn: issuedCertificateArn } : undefined), // Covers the replication connect and a cold task's first snapshot. healthCheckGracePeriod: "120 seconds", @@ -157,14 +155,12 @@ export const createMapleElectric = ({ tags, }) - // The public name, proxied through Cloudflare to the ALB. - if (domains.electric) { - yield* publishProxiedCname({ - id: "electric-public-cname", - hostname: domains.electric, - serviceUrl: service.url, - }) - } + // The public name, proxied through Cloudflare to the shared ALB. + yield* publishProxiedCname({ + id: "electric-public-cname", + hostname, + serviceUrl: service.url, + }) return { serviceUrl: service.url } }) diff --git a/apps/ingest/alchemy.run.ts b/apps/ingest/alchemy.run.ts index 5b6b8b4c9..543085424 100644 --- a/apps/ingest/alchemy.run.ts +++ b/apps/ingest/alchemy.run.ts @@ -135,7 +135,7 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbRole }: C // With an ingest domain the ALB terminates TLS on 443 behind Cloudflare's proxy, and // admits only Cloudflare's edge: that is what makes `Cf-IPCountry` trustworthy. A stage - // without one (PR previews) gets alchemy's default HTTP listener on 80, open to all. + // without one (PR previews) gets a plain HTTP listener on 80, open to all. // The group's `description` must not change: AWS treats it as immutable (a replace). const listenerPort = domains.ingest ? 443 : 80 const albSources = domains.ingest @@ -370,6 +370,32 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbRole }: C tags, }) + // The region's only ALB: `apps/electric` hangs a host rule and its own SNI certificate + // off this listener rather than paying for a second ALB. Ingest's rule is the catch-all. + const loadBalancer = yield* AWS.ELBv2.LoadBalancer("ingest-lb", { + type: "application", + scheme: "internet-facing", + subnets: network.publicSubnetIds, + securityGroups: [albSecurityGroup.groupId], + tags, + }) + // `certificateArn`, not `certificates`: the declarative list would strip electric's + // `ListenerCertificate` on every ingest deploy. + const listener = yield* AWS.ELBv2.Listener("ingest-listener", { + loadBalancerArn: loadBalancer, + port: listenerPort, + protocol: issuedCertificateArn ? "HTTPS" : "HTTP", + ...(issuedCertificateArn ? { certificateArn: issuedCertificateArn } : undefined), + defaultActions: [ + { + type: "fixedResponse", + statusCode: "404", + contentType: "text/plain", + messageBody: "Not Found", + }, + ], + }) + // Durability tier for the WAL (`apps/ingest/src/wal_store.rs`): sealed, // unexported segments, claimed by the next task if their owner dies. // Named up front so the env var below is a plain string. @@ -475,12 +501,15 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbRole }: C subnets: network.publicSubnetIds, securityGroups: [albSecurityGroup.groupId], - public: true, - // `port` is the CONTAINER port; the listener defaults to 443 with a - // certificate. Do not set `listenerPort`: Cloudflare cannot proxy to 3474. + // `port` is the CONTAINER port. The explicit `forward` names a fresh target group: an + // ALB target group belongs to one load balancer, so the owned-ALB one can't move here. port: INGEST_PORT, + loadBalancer: { + listener, + // Last, so electric's host rule matches first. + rules: [{ forward: `${INGEST_PORT}/http`, priority: 50000 }], + }, healthCheckPath: "/health", - ...(issuedCertificateArn ? { certificateArn: issuedCertificateArn } : undefined), // Covers the startup Postgres probe, which exits the process on failure. healthCheckGracePeriod: "60 seconds", // Old tasks stay scale-in protected for up to 15 minutes while the WAL drains, @@ -587,6 +616,9 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbRole }: C // Shared with `apps/electric`: two `AWS.EC2.Network`s in one stack fight // over the internet gateway. network, + // Shared with `apps/electric`, which routes by host on this listener. + listener, + albSecurityGroupId: albSecurityGroup.groupId, // Resolvable only inside the VPC; surfaced so a preview's logs say where the gateway points. collectorEndpoint, }