Skip to content

chore(infra): route electric through ingest's ALB instead of its own #1266

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Makisuo wants to merge 1 commit into
base: main
Choose a base branch
from
+80 −50
Open

chore(infra): route electric through ingest's ALB instead of its own #1266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions alchemy.run.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ export default Alchemy.Stack(
yield* serveWorker("api", api)

// Not wired into electric-sync: it reads `ELECTRIC_URL` from the secret store, so
// cutover is separate. Electric runs in ingest's VPC, hence `ingest &&`.
// cutover is separate. Electric runs in ingest's VPC behind its ALB (a host rule), hence `ingest &&`.
// Id must not be `"electric"` (the ECS service's id): alchemy keys state by id alone.
const electricDbRole =
db && profile.deploys.electric
Expand All @@ -259,13 +259,15 @@ export default Alchemy.Stack(
})
: undefined
const electric =
ingest && electricDbRole
ingest && electricDbRole && domains.electric
? yield* createMapleElectric({
stage,
region,
domains,
profile,
network: ingest.network,
listener: ingest.listener,
albSecurityGroupId: ingest.albSecurityGroupId,
hostname: domains.electric,
dbRole: electricDbRole,
})
: undefined
Expand Down
80 changes: 38 additions & 42 deletions apps/electric/alchemy.run.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,55 +14,42 @@
/** Absolute: alchemy has changed how a relative `dockerfile` resolves between releases. */
const DOCKERFILE = resolve("apps/electric/Dockerfile")

export interface CreateMapleElectricOptions extends Pick<
MapleStackContext,
"stage" | "region" | "domains" | "profile"
> {
export interface CreateMapleElectricOptions extends Pick<MapleStackContext, "stage" | "region" | "profile"> {
/** The ingest VPC: a second `AWS.EC2.Network` in one stack fights over the internet gateway. */
network: Pick<AWS.EC2.Network, "vpcId" | "publicSubnetIds">
/** Ingest's ALB listener, shared: a dedicated ALB costs more than this service's traffic. */
listener: AWS.ELBv2.Listener
/** The shared ALB's group, the only source admitted to ELECTRIC_PORT. */
albSecurityGroupId: AWS.EC2.SecurityGroup["groupId"]
/** Routed by host on the shared listener, so required (prd is the only stage that deploys this). */
hostname: string
/** The replication role on the instance's branch (`withReplication`), minted by the root. */
dbRole: Planetscale.PostgresRole
}

/**
* Self-hosted ElectricSQL on ECS Fargate, the upstream behind `apps/electric-sync`.
* Shares ingest's VPC but has its own cluster, ALB, security groups and certificate.
* Shares ingest's VPC and ALB (a host rule plus its own SNI certificate); own cluster and task group.
* Runbook: `docs/electric-sync.md`.

Check notice on line 33 in apps/electric/alchemy.run.ts

View check run for this annotation

Maple Review Bot / Maple / review

maintainability: Runbook this file points to still documents electric's own ALB and group 

The header comment still points at `docs/electric-sync.md` as the runbook, but that doc (lines 176-179) still describes Electric "with its own cluster, ALB, security groups and certificate inside the ingest fleet's VPC", and `docs/infra.md`'s `ECS.Service` pitfall note still explains the `certificateArn`/`listenerPort` pairing this diff removes. The next operator or on-call following it will reason about ALBs and a security group that no longer exist, including the manual `electric-lb` cleanup the deploy notes call out. Update those two sections with the shared listener, the host rule and the Cloudflare-only group.
*/
export const createMapleElectric = ({
stage,
region,
domains,
profile,
network,
listener,
albSecurityGroupId,
hostname,
dbRole,
}: CreateMapleElectricOptions) =>
Effect.gen(function* () {
const { taskSize, dbPoolSize } = profile.electric
const name = (base: string) => resolveAwsResourceName(base, stage, region)
const tags = { Service: "maple-electric", Region: region }

// Alchemy keys state by logical id: renaming these ids replaces live groups, and a
// new group must also get a new `groupName` or it collides with the old one.
// `securityGroups` apply to both ALB and tasks, so only the ALB's group may reach
// ELECTRIC_PORT; otherwise a task's public IP serves plaintext around the cert.
const listenerPort = domains.electric ? 443 : 80
const albSecurityGroup = yield* AWS.EC2.SecurityGroup("electric-lb-sg", {
vpcId: network.vpcId,
groupName: name("electric-lb"),
description: `Maple ElectricSQL - public ${listenerPort === 443 ? "HTTPS" : "HTTP"} to the load balancer`,
ingress: [
{
ipProtocol: "tcp",
fromPort: listenerPort,
toPort: listenerPort,
// Not narrowed to Cloudflare ranges (they rotate); ELECTRIC_SECRET authorizes.
cidrIpv4: "0.0.0.0/0",
description: "Shape requests from the electric-sync Worker",
},
],
})

// Only the shared ALB may reach ELECTRIC_PORT; otherwise a task's public IP serves
// plaintext around the cert. The ALB itself admits only Cloudflare (ingest's group),
// which is fine: electric-sync reaches this through the proxied hostname.
const taskSecurityGroup = yield* AWS.EC2.SecurityGroup("electric-task-sg", {
vpcId: network.vpcId,
groupName: name("electric-task"),
Expand All @@ -72,7 +59,7 @@
ipProtocol: "tcp",
fromPort: ELECTRIC_PORT,
toPort: ELECTRIC_PORT,
referencedGroupId: albSecurityGroup.groupId,
referencedGroupId: albSecurityGroupId,
description: "ALB to task",
},
],
Expand All @@ -91,12 +78,19 @@
// Shared with the electric-sync Worker; rotate by redeploying this first, then the Worker.
const apiSecret = yield* secret("api-secret", yield* requiredPlain("ELECTRIC_SECRET"))

const issuedCertificateArn = yield* issueRegionalCertificate({
const certificateArn = yield* issueRegionalCertificate({
id: "electric-cert",
hostname: domains.electric,
hostname,
region: resolveAwsRegion(region),
tags,
})
// SNI: the listener's default certificate is ingest's.
if (certificateArn) {
yield* AWS.ELBv2.ListenerCertificate("electric-listener-cert", {
listenerArn: listener,
certificateArn,
})
}

const baseEnv = {
ELECTRIC_PORT: String(ELECTRIC_PORT),
Expand Down Expand Up @@ -133,15 +127,19 @@

vpcId: network.vpcId,
subnets: network.publicSubnetIds,
securityGroups: [albSecurityGroup.groupId, taskSecurityGroup.groupId],
securityGroups: [taskSecurityGroup.groupId],
assignPublicIp: true,

// Public: the caller is a Worker with no route into the VPC; ELECTRIC_SECRET guards it.
// `port` is the container port; the listener goes to 443 once `certificateArn` is set.
public: true,
// The explicit `forward` names a fresh target group: an ALB target group belongs to
// one load balancer, so the one from electric's former ALB can't move here.
port: ELECTRIC_PORT,
loadBalancer: {
listener,
// Ahead of ingest's catch-all (priority 50000).
rules: [{ host: hostname, forward: `${ELECTRIC_PORT}/http`, priority: 10 }],
},
healthCheckPath: "/v1/health",
...(issuedCertificateArn ? { certificateArn: issuedCertificateArn } : undefined),
// Covers the replication connect and a cold task's first snapshot.
healthCheckGracePeriod: "120 seconds",

Expand All @@ -157,14 +155,12 @@
tags,
})

// The public name, proxied through Cloudflare to the ALB.
if (domains.electric) {
yield* publishProxiedCname({
id: "electric-public-cname",
hostname: domains.electric,
serviceUrl: service.url,
})
}
// The public name, proxied through Cloudflare to the shared ALB.
yield* publishProxiedCname({
id: "electric-public-cname",
hostname,
serviceUrl: service.url,
})
Comment on lines +159 to +163

@devin-ai-integration devin-ai-integration Bot Oct 5, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Electric DNS outruns its TLS certificate

During ALB replacement, publishProxiedCname can repoint Electric before ListenerCertificate attaches its certificate. Both depend on listener, but neither the service nor DNS record depends on ListenerCertificate. Electric requests then fail TLS until attachment finishes.

Learn more

Alchemy orders resources through the Outputs referenced in their properties, not the order in which the factory yields them. The Electric certificate attachment and the ECS service both reference the shared listener, but the CNAME references only the service URL. On the initial ALB migration, the service can become ready and update DNS while the listener still offers only ingest's default certificate. Cloudflare connects to the Electric hostname over TLS and rejects the wrong origin certificate until the attachment completes.

Example: On a production redeploy, the new ALB serves the Electric target group at 12:00:00 and the Electric CNAME changes at 12:00:01. If the SNI certificate attaches at 12:00:15, requests to electric.maple.dev fail during those 14 seconds instead of continuing to reach the old ALB.

Recommended fix: Carry an Output from AWS.ELBv2.ListenerCertificate into the DNS record's dependency chain so the CNAME cannot reconcile before SNI attachment. Keep the certificate on the listener before switching the public name; validate with an initial ALB replacement plan.

Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.


return { serviceUrl: service.url }
})
42 changes: 37 additions & 5 deletions apps/ingest/alchemy.run.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbRole }: C

// With an ingest domain the ALB terminates TLS on 443 behind Cloudflare's proxy, and
// admits only Cloudflare's edge: that is what makes `Cf-IPCountry` trustworthy. A stage
// without one (PR previews) gets alchemy's default HTTP listener on 80, open to all.
// without one (PR previews) gets a plain HTTP listener on 80, open to all.
// The group's `description` must not change: AWS treats it as immutable (a replace).
const listenerPort = domains.ingest ? 443 : 80
const albSources = domains.ingest
Expand Down Expand Up @@ -370,6 +370,32 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbRole }: C
tags,
})

// The region's only ALB: `apps/electric` hangs a host rule and its own SNI certificate
// off this listener rather than paying for a second ALB. Ingest's rule is the catch-all.
const loadBalancer = yield* AWS.ELBv2.LoadBalancer("ingest-lb", {
type: "application",
scheme: "internet-facing",
subnets: network.publicSubnetIds,
securityGroups: [albSecurityGroup.groupId],
tags,
})
// `certificateArn`, not `certificates`: the declarative list would strip electric's
// `ListenerCertificate` on every ingest deploy.
const listener = yield* AWS.ELBv2.Listener("ingest-listener", {
loadBalancerArn: loadBalancer,
port: listenerPort,
protocol: issuedCertificateArn ? "HTTPS" : "HTTP",
...(issuedCertificateArn ? { certificateArn: issuedCertificateArn } : undefined),
defaultActions: [
{
type: "fixedResponse",
statusCode: "404",
contentType: "text/plain",
messageBody: "Not Found",
},
],
})

// Durability tier for the WAL (`apps/ingest/src/wal_store.rs`): sealed,
// unexported segments, claimed by the next task if their owner dies.
// Named up front so the env var below is a plain string.
Expand Down Expand Up @@ -475,12 +501,15 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbRole }: C
subnets: network.publicSubnetIds,
securityGroups: [albSecurityGroup.groupId],

public: true,
// `port` is the CONTAINER port; the listener defaults to 443 with a
// certificate. Do not set `listenerPort`: Cloudflare cannot proxy to 3474.
// `port` is the CONTAINER port. The explicit `forward` names a fresh target group: an
// ALB target group belongs to one load balancer, so the owned-ALB one can't move here.
port: INGEST_PORT,
loadBalancer: {
listener,
// Last, so electric's host rule matches first.
rules: [{ forward: `${INGEST_PORT}/http`, priority: 50000 }],
},
healthCheckPath: "/health",
...(issuedCertificateArn ? { certificateArn: issuedCertificateArn } : undefined),
// Covers the startup Postgres probe, which exits the process on failure.
healthCheckGracePeriod: "60 seconds",
// Old tasks stay scale-in protected for up to 15 minutes while the WAL drains,
Expand Down Expand Up @@ -587,6 +616,9 @@ export const createMapleIngest = ({ stage, region, domains, profile, dbRole }: C
// Shared with `apps/electric`: two `AWS.EC2.Network`s in one stack fight
// over the internet gateway.
network,
// Shared with `apps/electric`, which routes by host on this listener.
listener,
albSecurityGroupId: albSecurityGroup.groupId,
// Resolvable only inside the VPC; surfaced so a preview's logs say where the gateway points.
collectorEndpoint,
}
Expand Down
Loading