From c0b1e20fc3167f9f118060b46d6832bf11808b48 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 1 Nov 2026 13:01:59 +0100 Subject: [PATCH 1/3] Postgres roles, a dispatcher that fits its memory, and faster cloud deploys A PostgresDatabase makes the roles a program needs (`roles`): each is granted to `app` for `set role`, a `login` one signs in through the pooler with a password of its own, handed out once like `app`'s, and one may bypass row-level security. A role taken off the list loses what it gave. Every password the node holds follows one model: readable until taken (by a run, `weft infra env`, or the card's copy button), then handed over until Reset mints a new one. Infra cards say so with `taken` and `handedOver`, which `weft infra env` reads instead of guessing. The cloud dispatcher keeps one byte-bounded cache of programs and caps the build files Cloud Run holds in memory; each role's memory, holders included, is a repository variable the install sets on every run. The install stores the CLI it used and deploys download it first; the first stock project after an update runs on the standard worker the install built. A node can hand a stored file to its project's own container (`container_link`). A local install's internal port listens on loopback and on the address Docker gives containers, asked on every start, never on every interface. launchctl failures are reported whole, and the runtime's stdout and stderr follow its log across a roll. Tangle's specialists may run read-only grep, rg, ffprobe, ffmpeg into their scratch folder, a read-only psql transaction and `weft stop`; the matchers refuse every shell construct they cannot read the way bash and zsh do. --- .github/workflows/ci.yml | 14 +- .github/workflows/install-gcp.yml | 58 +- CHANGELOG.md | 65 ++ Cargo.lock | 1 + .../database/images/credential/bootstrap.py | 436 +++++++++---- .../images/credential/test_bootstrap.py | 199 +++++- catalog/postgres/database/metadata.json | 11 +- catalog/postgres/database/mod.rs | 148 ++++- catalog/postgres/database/tests.rs | 101 ++- crates/weft-broker/src/runtime_storage.rs | 126 +++- crates/weft-broker/src/state.rs | 7 + crates/weft-broker/tests/db/install_cli.rs | 45 ++ crates/weft-broker/tests/db/main.rs | 1 + crates/weft-broker/tests/db/old_shapes.rs | 13 +- crates/weft-cli/src/commands/daemon.rs | 96 ++- crates/weft-cli/src/commands/infra_card.rs | 9 +- crates/weft-cli/src/commands/infra_env.rs | 105 ++- crates/weft-cli/templates/ci/gcp.yml | 15 +- crates/weft-core/src/content_cache.rs | 101 ++- crates/weft-core/src/context.rs | 28 +- crates/weft-core/src/context/pure.rs | 7 +- crates/weft-core/src/install.rs | 44 ++ crates/weft-core/src/live.rs | 72 +- crates/weft-core/src/node_test.rs | 10 +- crates/weft-core/src/storage/mod.rs | 19 +- crates/weft-dispatcher/src/api/install_cli.rs | 61 ++ crates/weft-dispatcher/src/api/mod.rs | 35 +- crates/weft-dispatcher/src/app.rs | 5 +- .../weft-dispatcher/src/build/blob_cache.rs | 26 +- crates/weft-dispatcher/src/build/mod.rs | 31 +- crates/weft-dispatcher/src/build/prune.rs | 8 +- crates/weft-dispatcher/src/search_index.rs | 36 +- crates/weft-dispatcher/src/state.rs | 46 +- crates/weft-dispatcher/src/store/postgres.rs | 15 +- crates/weft-dispatcher/tests/db/db_build.rs | 26 +- .../tests/db/db_maintenance.rs | 2 +- crates/weft-dispatcher/tests/resync_memory.rs | 284 ++++++++ .../images/container_fetch/Dockerfile | 12 + .../images/container_fetch/server.py | 51 ++ .../nodes/container_fetch/metadata.json | 30 + .../nodes/container_fetch/mod.rs | 50 ++ .../fixtures/container_link/src/main.weft | 12 + .../fixtures/container_link/weft.toml | 4 + crates/weft-e2e/tests/e2e/container_link.rs | 28 + crates/weft-e2e/tests/e2e/main.rs | 1 + crates/weft-e2e/tests/e2e/postgres.rs | 128 ++++ crates/weft-engine/src/storage.rs | 1 + crates/weft-platform-local/src/docker.rs | 74 +++ crates/weft-platform-traits/src/config.rs | 30 +- .../weft-platform-traits/src/object_store.rs | 12 +- crates/weft-platform-traits/src/roles.rs | 11 + crates/weft-runtime/Cargo.toml | 3 + crates/weft-runtime/src/lib.rs | 33 +- crates/weft-runtime/src/log_file.rs | 55 +- deploy/terraform/gcp/outputs.tf | 5 + deploy/terraform/gcp/serverless.tf | 30 + deploy/terraform/gcp/variables.tf | 43 +- docs/src/nodes/storage.md | 3 +- docs/src/running/cloud.md | 25 +- packages/weft-graph/src/protocol.ts | 10 + .../lib/components/project/ProjectNode.svelte | 22 +- .../lib/components/ui/CopyButton.svelte | 56 +- scripts/release-cli.sh | 127 +++- scripts/release-cli.test.sh | 130 ++++ tangle/claude-code/.claude/agents/designer.md | 2 +- .../.claude/agents/frontend-builder.md | 4 +- .../.claude/agents/group-builder.md | 4 +- .../claude-code/.claude/agents/node-smith.md | 2 +- .../.claude/agents/prompt-engineer.md | 2 +- .../.claude/hooks/specialist_commands.json | 50 +- .../.claude/hooks/specialist_commands.py | 589 ++++++++++++++++- .../skills/weft-cloud-install/SKILL.md | 10 + .../.claude/skills/weft-frontend/SKILL.md | 2 +- .../.claude/skills/weft-incidents/SKILL.md | 9 + tangle/claude-code/README.md | 2 +- .../tests/test_specialist_commands.py | 286 ++++++++ .../.cline/skills/weft-cloud-install/SKILL.md | 10 + .../.cline/skills/weft-frontend/SKILL.md | 2 +- .../.cline/skills/weft-incidents/SKILL.md | 9 + .../skills/weft-cloud-install/SKILL.md | 10 + .../.agents/skills/weft-incidents/SKILL.md | 9 + tangle/codex/.codex/agents/designer.toml | 2 +- .../codex/.codex/agents/frontend-builder.toml | 4 +- tangle/codex/.codex/agents/group-builder.toml | 4 +- tangle/codex/.codex/agents/node-smith.toml | 2 +- .../codex/.codex/agents/prompt-engineer.toml | 2 +- .../.codex/hooks/specialist_commands.json | 123 +++- .../codex/.codex/hooks/specialist_commands.py | 447 ++++++++++++- tangle/codex/README.md | 2 +- .../codex/tests/test_specialist_commands.py | 247 +++++++ .../skills/weft-cloud-install/SKILL.md | 10 + .../.cursor/skills/weft-frontend/SKILL.md | 2 +- .../.cursor/skills/weft-incidents/SKILL.md | 9 + .../skills/weft-cloud-install/SKILL.md | 10 + .../.windsurf/skills/weft-incidents/SKILL.md | 9 + .../skills/weft-cloud-install/SKILL.md | 10 + .../.agents/skills/weft-frontend/SKILL.md | 2 +- .../.agents/skills/weft-incidents/SKILL.md | 9 + .../skills/weft-cloud-install/SKILL.md | 10 + .../.gemini/skills/weft-frontend/SKILL.md | 2 +- .../.gemini/skills/weft-incidents/SKILL.md | 9 + .../skills/weft-cloud-install/SKILL.md | 10 + .../.github/skills/weft-frontend/SKILL.md | 2 +- .../.github/skills/weft-incidents/SKILL.md | 9 + .../.junie/skills/weft-cloud-install/SKILL.md | 10 + .../.junie/skills/weft-frontend/SKILL.md | 2 +- .../.junie/skills/weft-incidents/SKILL.md | 9 + tangle/kilo-code/.kilo/agent/designer.md | 2 +- .../kilo-code/.kilo/agent/frontend-builder.md | 4 +- tangle/kilo-code/.kilo/agent/group-builder.md | 4 +- tangle/kilo-code/.kilo/agent/node-smith.md | 2 +- .../kilo-code/.kilo/agent/prompt-engineer.md | 2 +- .../.kilo/lib/specialist-commands.ts | 616 ++++++++++++++++- .../.kilo/skills/weft-cloud-install/SKILL.md | 10 + .../.kilo/skills/weft-frontend/SKILL.md | 2 +- .../.kilo/skills/weft-incidents/SKILL.md | 9 + .../kilo-code/.kilo/specialist_commands.json | 139 +++- tangle/kilo-code/README.md | 2 +- .../tests/specialist-commands.test.ts | 258 +++++++- tangle/opencode/.opencode/agents/designer.md | 2 +- .../.opencode/agents/frontend-builder.md | 4 +- .../.opencode/agents/group-builder.md | 4 +- .../opencode/.opencode/agents/node-smith.md | 2 +- .../.opencode/agents/prompt-engineer.md | 2 +- tangle/opencode/.opencode/agents/tangle.md | 2 +- .../skills/weft-cloud-install/SKILL.md | 10 + .../.opencode/skills/weft-frontend/SKILL.md | 2 +- .../.opencode/skills/weft-incidents/SKILL.md | 9 + .../.opencode/specialist-commands/decide.ts | 617 +++++++++++++++++- .../specialist_commands.json | 49 +- tangle/opencode/README.md | 10 +- .../tests/specialist-commands.test.ts | 256 +++++++- 132 files changed, 6706 insertions(+), 564 deletions(-) create mode 100644 crates/weft-broker/tests/db/install_cli.rs create mode 100644 crates/weft-dispatcher/src/api/install_cli.rs create mode 100644 crates/weft-dispatcher/tests/resync_memory.rs create mode 100644 crates/weft-e2e/fixtures/container_link/nodes/container_fetch/images/container_fetch/Dockerfile create mode 100644 crates/weft-e2e/fixtures/container_link/nodes/container_fetch/images/container_fetch/server.py create mode 100644 crates/weft-e2e/fixtures/container_link/nodes/container_fetch/metadata.json create mode 100644 crates/weft-e2e/fixtures/container_link/nodes/container_fetch/mod.rs create mode 100644 crates/weft-e2e/fixtures/container_link/src/main.weft create mode 100644 crates/weft-e2e/fixtures/container_link/weft.toml create mode 100644 crates/weft-e2e/tests/e2e/container_link.rs create mode 100755 scripts/release-cli.test.sh create mode 100644 tangle/claude-code/tests/test_specialist_commands.py create mode 100644 tangle/codex/tests/test_specialist_commands.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba02cf157..ea545427d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,6 +175,9 @@ jobs: # What the install workflow's targeted applies add for every # `moved` block, read from the files above. - run: scripts/terraform-moved-targets.test.sh + # Where a deploy finds its CLI: the install, then weft's release, + # then a build, with a fake web standing in for both. + - run: scripts/release-cli.test.sh # The name the main ruleset requires: green only when every Rust check # above is, and the Terraform the install runs validates. It runs whatever happened above (unless the run was @@ -290,9 +293,11 @@ jobs: # The container's pure parts, on the standard library alone. - run: python3 catalog/postgres/database/images/credential/test_bootstrap.py - # Kilo Code's Tangle plugin (`tangle/kilo-code/`): its tests are - # TypeScript run straight by `node --test`, which strips the types - # itself from Node 22.18 on, so they need no install either. + # The Tangle plugins and hooks that decide what a specialist may run: + # Kilo Code's and OpenCode's tests are TypeScript run straight by + # `node`, which strips the types itself from Node 22.18 on, and Claude + # Code's and Codex's are Python on the standard library, so none of + # them needs an install. tangle: name: Tangle plugin tests runs-on: ubuntu-latest @@ -302,6 +307,9 @@ jobs: with: node-version: 24 - run: node --test tangle/kilo-code/tests/*.test.ts + - run: node tangle/opencode/tests/specialist-commands.test.ts + - run: python3 tangle/claude-code/tests/test_specialist_commands.py + - run: python3 tangle/codex/tests/test_specialist_commands.py # The browser extension ships to three stores gated only on a version # bump, so its checks belong in the same gate as everything else JS. diff --git a/.github/workflows/install-gcp.yml b/.github/workflows/install-gcp.yml index aa335564f..6517a7078 100644 --- a/.github/workflows/install-gcp.yml +++ b/.github/workflows/install-gcp.yml @@ -20,6 +20,12 @@ # WEFT_BROKER_MAX_COPIES (the same for the broker; # 4 when unset). Each copy of those two takes an # equal share of the database's connections. +# WEFT_DISPATCHER_MEMORY, WEFT_BROKER_MEMORY, +# WEFT_LISTENER_MEMORY, WEFT_SUPERVISOR_MEMORY (the +# memory of each copy of that role, from 512Mi to +# 4Gi; 1Gi for the dispatcher and 512Mi for the +# others when unset), WEFT_HOLDER_MEMORY (each +# holder's, at least 512Mi, 512Mi when unset). # repository secrets WEFT_DATABASE_URL (the Postgres the install keeps # everything in; not needed with the variable # WEFT_DATABASE set to cloud_sql, which gives the @@ -95,13 +101,19 @@ jobs: # The optional sizes, handed to Terraform only when set: an unset # repository variable reads as an empty string, which Terraform - # refuses for a number, so an unset one leaves Terraform's default. + # refuses for a number or a memory size, so an unset one leaves + # Terraform's default. # SYNC: these TF_VAR_* names <-> deploy/terraform/gcp/variables.tf - name: the optional sizes env: CLOUD_SQL_MAX_CONNECTIONS: ${{ vars.WEFT_CLOUD_SQL_MAX_CONNECTIONS }} DISPATCHER_MAX_COPIES: ${{ vars.WEFT_DISPATCHER_MAX_COPIES }} BROKER_MAX_COPIES: ${{ vars.WEFT_BROKER_MAX_COPIES }} + DISPATCHER_MEMORY: ${{ vars.WEFT_DISPATCHER_MEMORY }} + BROKER_MEMORY: ${{ vars.WEFT_BROKER_MEMORY }} + LISTENER_MEMORY: ${{ vars.WEFT_LISTENER_MEMORY }} + SUPERVISOR_MEMORY: ${{ vars.WEFT_SUPERVISOR_MEMORY }} + HOLDER_MEMORY: ${{ vars.WEFT_HOLDER_MEMORY }} run: | set_if_given() { if [ -n "$2" ]; then @@ -111,6 +123,11 @@ jobs: set_if_given TF_VAR_cloud_sql_max_connections "$CLOUD_SQL_MAX_CONNECTIONS" set_if_given TF_VAR_dispatcher_max_copies "$DISPATCHER_MAX_COPIES" set_if_given TF_VAR_broker_max_copies "$BROKER_MAX_COPIES" + set_if_given TF_VAR_dispatcher_memory "$DISPATCHER_MEMORY" + set_if_given TF_VAR_broker_memory "$BROKER_MEMORY" + set_if_given TF_VAR_listener_memory "$LISTENER_MEMORY" + set_if_given TF_VAR_supervisor_memory "$SUPERVISOR_MEMORY" + set_if_given TF_VAR_holder_memory "$HOLDER_MEMORY" - uses: actions/checkout@v7 @@ -210,12 +227,16 @@ jobs: exit 1 fi - # The runtime image and the worker builder base this checkout names, - # pushed to the install's registry. Each is taken from the install's - # registry if there, else copied from the release's (the same tag is - # the same contents), else built. A fork carrying its own changes - # hashes to new tags and gets exactly its own runtime, made from the - # `weft-runtime` built above instead of compiled again in Docker. + # The runtime image, the worker builder base and the standard worker + # this checkout names, pushed to the install's registry. Each is taken + # from the install's registry if there, else copied from the + # release's (the same tag is the same contents), else built. A fork + # carrying its own changes hashes to new tags and gets exactly its + # own runtime, made from the `weft-runtime` built above instead of + # compiled again in Docker. The standard worker is the one every + # project whose nodes are all the standard library's runs, so the + # first of them after this update finds it ready and builds nothing + # (crates/weft-dispatcher/src/build/mod.rs, `built_whole`). - name: build the runtime's images env: BUILT_HERE: ${{ steps.release.outputs.found != 'yes' }} @@ -426,6 +447,29 @@ jobs: gcloud secrets versions access latest --secret "$key_secret" --project "${{ vars.GCP_PROJECT_ID }}" \ | "$GITHUB_WORKSPACE/target/release/weft" catalog preload --dispatcher "$url" --key-stdin + # The CLI this run used (the release's for this exact source, or the + # one built above), stored in the install under that source's tree, so + # a project's deploy downloads it (`GET /install/cli/`, through + # scripts/release-cli.sh) instead of compiling it: a fork's CLI exists + # nowhere else. The deploy runs on the same runner as this job + # (ubuntu-24.04, x86_64), so this binary runs there. The sha256 goes + # last, since it is what says the CLI is whole, and only this source's + # CLI is kept: once the install runs it, no deploy asks for another. + # This workflow's account writes the bucket as the project's Owner. + # SYNC: install/cli//{weft,sha256} <-> crates/weft-core/src/install.rs (install_cli_keys) + # SYNC: runs-on <-> crates/weft-cli/templates/ci/gcp.yml (runs-on) + - name: store the weft CLI in the install + working-directory: deploy/terraform/gcp + run: | + bucket=$(terraform output -raw files_bucket) + tree=$(git -C "$GITHUB_WORKSPACE" rev-parse 'HEAD^{tree}') + cli="$GITHUB_WORKSPACE/target/release/weft" + sha256sum "$cli" | cut -d' ' -f1 > "$RUNNER_TEMP/cli.sha256" + gcloud storage cp "$cli" "gs://$bucket/install/cli/$tree/weft" + gcloud storage cp "$RUNNER_TEMP/cli.sha256" "gs://$bucket/install/cli/$tree/sha256" + gcloud storage ls "gs://$bucket/install/cli/" | { grep -v "/install/cli/$tree/\$" || true; } \ + | xargs -r gcloud storage rm --recursive + # Written to the run's log as well as its summary: `gh run view` # shows the log, never the summary, and whoever installs from a # terminal needs the address and the key command from there. diff --git a/CHANGELOG.md b/CHANGELOG.md index e25960608..f7b5f01b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,71 @@ What changed between versions of weft that you need to know about when you update. To update, run `weft maintenance on`, pull and run `./setup.sh` again, then run `weft maintenance off`. +## Unreleased + +**A PostgresDatabase makes the roles your program needs.** Since the program +signs in as `app`, which is no superuser, a database that kept tenants apart +with roles of its own could no longer make them, `set role` to them, or let a +site sign in as one, and a setup step writing past a `FORCE` row-level +security policy was refused. List them in the node's new `roles` input: on +every start the database makes each one, grants it to `app` (so `set role` +works), lets one marked `login` sign in through the pooler with a password of +its own, handed out once like the program's (`weft infra env --into +.env --set SITE_PASSWORD=" password"` takes it, and its Reset button on +the card mints a new one), and lets one marked +`bypassRowLevelSecurity` write past every policy. Roles you made by hand keep +what they own; list them under the same names and the database takes them +over, except that a `login` role gets a password of its own, so whatever signs +in as it needs the new one from the card. Taking a role off the list takes back +what it gave: `app` can no longer `set role` to it, and it signs in and passes +row-level security no more; the role and what it owns stay. Changing the list +restarts the database. + +**A cloud dispatcher fits in its memory, and keeps the memory you give it.** +On Cloud Run a role's files are held in its memory, and the dispatcher kept +up to 1GiB of a build's files there, and 64 programs whatever their size; +past 512Mi a resync failed with `Service Unavailable`. It now keeps at most +64MiB of those files there and about 90MiB of programs. The dispatcher gets +1Gi by default, and each role's memory is a repository variable on your fork +(`WEFT_DISPATCHER_MEMORY`, `WEFT_BROKER_MEMORY`, `WEFT_LISTENER_MEMORY`, +`WEFT_SUPERVISOR_MEMORY`): a value set by hand in the Cloud Run console used +to be put back to 512Mi by every install run, and now the install sets yours. + +**A fork's deploy no longer compiles the weft CLI.** The install workflow now +keeps the CLI it used in your install, and the deploy workflow downloads it +from there first, checking its sha256, before weft's release and before +compiling it. Run `weft ci add` again in each project to get the new deploy +workflow, and the install workflow once, to store the CLI. + +**The first project of standard nodes after a cloud update starts at once.** +The install already built the standard worker, and that project built it +again, waiting about three minutes; it now runs on the one the install built. + +**A node can hand a stored file to its project's own container.** On a local +install an `internal_link` points at `127.0.0.1`, which inside a container is +the container itself, so a download from there failed. A node's code now asks +for `container_link(&file)`, which a project's own container opens on any +install. On Linux, weft's internal port no longer listens on every address of +your machine: only on loopback and on the one address Docker gives containers +for it, which weft asks Docker each time it starts. + +**A database's password is handed over once to whoever takes it.** `weft infra +env` writing it into a file, or the card's copy button, now hands it over the +way a run storing it does: the card shows it as handed over from then on, and +its Reset button mints a new one. + +**`weft infra env --as` names the secret it writes.** On a card with more than +one secret (a database with a `login` role), `--as` used to take whichever one +still showed its value; it now refuses and lists them, and `--set +NAME=Label` picks one. + +**`weft daemon start` on a Mac says why launchd refused.** The error now +carries everything `launchctl` printed and its exit status, where it used to +end at `launchctl bootstrap failed:`. What fails before the runtime even +starts (a secrets file it cannot read, a binary gone) now lands in the +install's log, and so does a runtime's panic after its log has rolled over, +which used to go to a file nobody reads. + ## 2026-11-01 (39a50c84) **Triggers come back after an update again.** Since 0.2.281, the triggers diff --git a/Cargo.lock b/Cargo.lock index 4a76985dd..174bb2b48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5254,6 +5254,7 @@ dependencies = [ "clap", "futures", "hex", + "libc", "parking_lot 0.13.0", "reqwest 0.13.5", "serde_json", diff --git a/catalog/postgres/database/images/credential/bootstrap.py b/catalog/postgres/database/images/credential/bootstrap.py index 808e1315a..31e9179e2 100644 --- a/catalog/postgres/database/images/credential/bootstrap.py +++ b/catalog/postgres/database/images/credential/bootstrap.py @@ -12,7 +12,9 @@ up for the program (`converge`): the program's own role, which owns the program's database and everything in it and is no superuser, signing in with the password on the shared volume; - then it tells the pooler who may sign in. Every start does it + the roles the node lists, each granted to the program's role, a + login one with a password of its own beside the program's; then + it tells the pooler who may sign in. Every start does it again, so a database made before any of this existed is brought to the same shape. Until it is done, `/health` says not ready. @@ -29,14 +31,18 @@ It also serves the graph: `/live` shows the database and user to sign in as, whether the password is still readable, and who is connected (to Postgres and to the pooler, read when the card - asks, never on a timer); its one button, `/action` with + asks, never on a timer); its Reset button, `/action` with `reset_password`, mints a new password, sets it on the program's role over the unix socket (which needs no password), tells the pooler, and makes it readable again. Then it tells weft the new password (`WEFT_VALUES_URL`, the agent beside this unit), which writes it into the connection the node published, so the program's next run signs in with it; the card shows it until the - node's next run retires it. + node's next run, or a reader that copied it out (`password_taken`, + with the password as proof), retires it. Each login role's + password works the same way, under its own name + (`reset_password:`, `password_taken:`), up to telling + the pooler: weft keeps no connection for it. Where the files live, which ports and names to use, all come from the node that declares this container, so the two sides cannot drift. @@ -73,6 +79,10 @@ WATCH_DATABASE = os.environ.get("WEFT_WATCH_DATABASE") # The extensions the program's database needs, as a JSON list of names. EXTENSIONS_JSON = os.environ.get("WEFT_EXTENSIONS") +# The roles it needs beside the program's own, as a JSON list of +# `{"name", "login", "bypassRowLevelSecurity"}`, checked by the node. +# SYNC: WEFT_ROLES <-> catalog/postgres/database/mod.rs (Role) +ROLES_JSON = os.environ.get("WEFT_ROLES") # The Postgres this unit runs, the first major that closes idle sessions, # and what to warn of when it is older (empty when it is not). POSTGRES_MAJOR = os.environ.get("WEFT_POSTGRES_MAJOR") @@ -91,7 +101,8 @@ "WEFT_SECRET_DIR": SECRET_DIR, "WEFT_PASSWORD_FILE": PASSWORD_FILE, "WEFT_CREDENTIAL_PORT": PORT, "WEFT_SOCKET_DIR": SOCKET_DIR, "WEFT_POSTGRES_PORT": POSTGRES_PORT, "WEFT_ADMIN_USER": ADMIN_USER, "WEFT_APP_USER": APP_USER, "WEFT_DATABASE": DATABASE, "WEFT_POOLER_DATABASE": POOLER_DATABASE, - "WEFT_WATCH_DATABASE": WATCH_DATABASE, "WEFT_EXTENSIONS": EXTENSIONS_JSON, "WEFT_POSTGRES_MAJOR": POSTGRES_MAJOR, + "WEFT_WATCH_DATABASE": WATCH_DATABASE, "WEFT_EXTENSIONS": EXTENSIONS_JSON, "WEFT_ROLES": ROLES_JSON, + "WEFT_POSTGRES_MAJOR": POSTGRES_MAJOR, "WEFT_CLOSES_IDLE_SESSIONS_FROM": CLOSES_IDLE_SESSIONS_FROM, "WEFT_DATA_DIR": DATA_DIR, "WEFT_IDLE_SESSION_SECONDS": IDLE_SESSION_SECONDS, "WEFT_POOLER_DIR": POOLER_DIR, "WEFT_POOLER_PORT": POOLER_PORT, "WEFT_POOLER_USERS_FILE": POOLER_USERS_FILE, @@ -108,7 +119,25 @@ EXTENSIONS = json.loads(EXTENSIONS_JSON) if not isinstance(EXTENSIONS, list) or not all(isinstance(name, str) and name for name in EXTENSIONS): sys.exit(f"WEFT_EXTENSIONS must be a JSON list of names, got {EXTENSIONS_JSON}") -SEALED_FILE = os.path.join(SECRET_DIR, "sealed") + + +def read_roles(raw: str) -> list[dict]: + """The node's roles, each with all three keys, or exit naming what is wrong.""" + roles = json.loads(raw) + keys = {"name", "login", "bypassRowLevelSecurity"} + if not isinstance(roles, list) or not all( + isinstance(role, dict) and set(role) == keys and isinstance(role["name"], str) and role["name"] + and isinstance(role["login"], bool) and isinstance(role["bypassRowLevelSecurity"], bool) + for role in roles + ): + sys.exit(f"WEFT_ROLES must be a JSON list of {sorted(keys)}, got {raw}") + return roles + + +ROLES = read_roles(ROLES_JSON) +# The roles that sign in by themselves, through the pooler, each with a +# password of its own: the program's, then the node's login roles. +SIGNS_IN = [APP_USER] + [role["name"] for role in ROLES if role["login"]] # SYNC: credential routes <-> catalog/postgres/database/mod.rs # CREDENTIAL_PATH / CREDENTIAL_STORED_PATH / HEALTH_PATH @@ -119,6 +148,15 @@ LIVE_PATH = "/live" ACTION_PATH = "/action" RESET_ACTION = "reset_password" +# What a reader presses once it copied a password out of the card (`weft +# infra env`, the editor's copy button), with the password as proof. +# SYNC: TAKEN_ACTION <-> crates/weft-core/src/live.rs (LiveItem::taken, taken_press), +# packages/weft-graph/src/webview/lib/components/project/ProjectNode.svelte (takenFrom) +TAKEN_ACTION = "password_taken" +# A login role's own button and its own taken press carry its name after a +# `:` (`reset_password:`), so each is pressed by its own name; the +# program's carry none. +ROLE_MARK = ":" # Set once `converge` finished: the program's role is in place and the # pooler knows who may sign in. `/health` answers ready from then on. @@ -129,8 +167,12 @@ NOT_READY = {"why": "the database is still being set up for the program"} # How long `converge` waits before trying again after the database refused. CONVERGE_RETRY_SECONDS = 30 -# One change of the password at a time: two presses at once would leave -# the pooler told one password and the database holding the other. +# One change of a password at a time: two presses at once would leave the +# pooler told one password and the database holding the other. Sealing one +# takes it too, comparing and sealing in one step, so a password a reset +# just minted is never sealed by a taker holding the one it replaced; a run +# confirming its password while a reset tells weft (up to 30s) waits for +# it, and then confirms against the new one. CHANGING = threading.Lock() @@ -147,7 +189,7 @@ def mint() -> None: os.makedirs(SECRET_DIR, mode=0o750, exist_ok=True) if os.path.exists(PASSWORD_FILE): return - write_password(new_password()) + write_password(APP_USER, new_password()) def too_old_for_a_new_database(closes_idle_sessions: bool, has_data: bool) -> str | None: @@ -192,12 +234,31 @@ def write_file(path: str, text: str) -> None: os.close(dir_fd) -def write_password(password: str) -> None: - write_file(PASSWORD_FILE, password) +def password_file(role: str) -> str: + """Where `role`'s password lives: the program's is the one Postgres was + made with, each login role's sits beside it. A role name is lowercase + letters, digits and `_` (the node checks), so it is a file name as it is.""" + return PASSWORD_FILE if role == APP_USER else os.path.join(SECRET_DIR, f"role.{role}") + + +def write_password(role: str, password: str) -> None: + write_file(password_file(role), password) + + +def sealed_file(role: str) -> str: + """The file whose being there says `role`'s password was handed over: + beside its password, for the program's role and each login role alike.""" + return os.path.join(SECRET_DIR, "sealed") if role == APP_USER else password_file(role) + ".sealed" -def sealed() -> bool: - return os.path.exists(SEALED_FILE) +def sealed(role: str) -> bool: + return os.path.exists(sealed_file(role)) + + +def seal(role: str) -> None: + """Mark `role`'s password handed over. Idempotent. Raises `OSError`.""" + with open(sealed_file(role), "w", encoding="utf-8") as f: + f.write("stored") class Refused(Exception): @@ -322,11 +383,45 @@ def rows(printed: str) -> list[dict]: ALTER DATABASE :"database" OWNER TO :"app"; """ + OWNED_BY_ADMIN -# Postgres closes the role's sessions once they sit idle, which is how a +# Postgres closes a role's sessions once they sit idle, which is how a # connection a client holds and does not use goes back to the pooler -# whoever the client is. Set on a Postgres that has the setting only: an -# older one keeps its idle sessions open, which its card warns of. -CLOSE_IDLE_SESSIONS = """ALTER ROLE :"app" SET idle_session_timeout = :'idle';\n""" +# whoever the client is. Set on each role that signs in, on a Postgres that +# has the setting only: an older one keeps its idle sessions open, which its +# card warns of. +CLOSE_IDLE_SESSIONS = """ALTER ROLE :"role" SET idle_session_timeout = :'idle';\n""" + + +def role_script(role: dict) -> str: + """One of the node's roles, made when missing and held to what its entry + says on every start, never a superuser and never one that makes roles, + and granted to the program's role so the program can `set role` to it. + The two attributes the entry sets are picked from fixed words, never + written from its values.""" + login = "LOGIN" if role["login"] else "NOLOGIN" + bypass = "BYPASSRLS" if role["bypassRowLevelSecurity"] else "NOBYPASSRLS" + return ( + "SELECT format('CREATE ROLE %I', :'role') WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'role')\n" + "\\gexec\n" + f'ALTER ROLE :"role" WITH NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION {login} {bypass};\n' + 'GRANT :"role" TO :"app";\n' + ) + + +# The roles the node listed once and lists no more: every role `app` was +# granted by the superuser (only `converge` grants that way) that the list no +# longer names, never one of Postgres's own. Each stays in the database with +# what it owns, and loses what the list gave it: `app`'s membership (so the +# program can no longer `set role` to it), signing in, and passing the +# row-level security policies. +STRIP_UNLISTED = r""" +SELECT format('REVOKE %I FROM %I', r.rolname, :'app'), format('ALTER ROLE %I WITH NOLOGIN NOBYPASSRLS', r.rolname) + FROM pg_auth_members m JOIN pg_roles r ON r.oid = m.roleid + WHERE m.member = (SELECT oid FROM pg_roles WHERE rolname = :'app') + AND m.grantor = (SELECT oid FROM pg_roles WHERE rolname = :'admin') + AND r.rolname !~ '^pg_' + AND r.rolname <> ALL (SELECT json_array_elements_text(:'listed'::json)) +\gexec +""" def scram_accepts(verifier: str | None, password: str) -> bool: @@ -348,35 +443,39 @@ def scram_accepts(verifier: str | None, password: str) -> bool: return False -def pooler_users(verifier: str) -> str: - """The pooler's file of who may sign in: the program's role with its - verifier (the pooler checks a client's password against it, and signs - in to Postgres with what the client proved), and the superuser with no +def pooler_users(verifiers: dict[str, str]) -> str: + """The pooler's file of who may sign in: each role that signs in with its + verifier (the pooler checks a client's password against it, and signs in + to Postgres with what the client proved), and the superuser with no password at all, which the pooler's own rules let in only to its admin console over its unix socket.""" quoted = lambda value: '"' + value.replace('"', '""') + '"' - return f"{quoted(APP_USER)} {quoted(verifier)}\n{quoted(ADMIN_USER)} \"\"\n" + lines = [f"{quoted(role)} {quoted(verifier)}\n" for role, verifier in verifiers.items()] + return "".join(lines) + f"{quoted(ADMIN_USER)} \"\"\n" -def app_verifier() -> str | None: - """The program's role's stored password verifier, or None when it has none.""" - found = rows(psql("SELECT rolpassword FROM pg_authid WHERE rolname = :'app';\n", {"app": APP_USER})) +def verifier(role: str) -> str | None: + """`role`'s stored password verifier, or None when it has none.""" + found = rows(psql("SELECT rolpassword FROM pg_authid WHERE rolname = :'role';\n", {"role": role})) if not found: return None return found[0]["rolpassword"] or None -def set_app_password(password: str) -> None: - psql("ALTER ROLE :\"app\" PASSWORD :'password';\n", {"app": APP_USER, "password": password}) +def set_password(role: str, password: str) -> None: + psql("ALTER ROLE :\"role\" PASSWORD :'password';\n", {"role": role, "password": password}) def tell_the_pooler() -> None: """Write who may sign in from the database's own records, then have the pooler read it again. Raises `Refused` when the pooler would not.""" - verifier = app_verifier() - if verifier is None: - raise Refused(f"the role {APP_USER} has no password in the database") - write_file(POOLER_USERS_FILE, pooler_users(verifier)) + verifiers = {} + for role in SIGNS_IN: + held = verifier(role) + if held is None: + raise Refused(f"the role {role} has no password in the database") + verifiers[role] = held + write_file(POOLER_USERS_FILE, pooler_users(verifiers)) psql("RELOAD;\n", {}, pooler=True) @@ -431,25 +530,51 @@ def converge() -> None: sys.stderr.write(f"credential: the program signs in as {APP_USER}, through the pooler\n") +def signs_in_with_its_password(role: str, password: str) -> None: + """Give `role` `password` when it does not already have it, and close its + idle sessions. Raises `Refused`.""" + if CLOSES_IDLE_SESSIONS: + psql(CLOSE_IDLE_SESSIONS, {"role": role, "idle": f"{IDLE_SESSION_SECONDS}s"}) + if not scram_accepts(verifier(role), password): + set_password(role, password) + + def converge_once() -> str | None: """One pass of `converge`: what the database refused, or None once done. - The password file is the one source of the program's password: the - role is given it when it does not already have it, so a reset that - stopped halfway is settled by the next start. Each extension the - node lists is made by the superuser when it is not there yet.""" + Each password file is the one source of its role's password: the role + is given it when it does not already have it, so a reset that stopped + halfway is settled by the next start. The program's is minted by the + init container, before Postgres starts; a login role's is minted here + the first time it is listed, and kept from then on. Each role and each + extension the node lists is made by the superuser when it is not there + yet, and each role it listed once and no longer does loses what the list + gave it (`STRIP_UNLISTED`).""" wait_until("Postgres to accept connections", postgres_not_serving) - password = stored_password() + password = stored_password(APP_USER) if password is None: sys.exit("credential: no password on the shared volume; the init container writes it before this one starts") try: psql(CONVERGE, {"app": APP_USER, "admin": ADMIN_USER, "database": DATABASE}) - if CLOSES_IDLE_SESSIONS: - psql(CLOSE_IDLE_SESSIONS, {"app": APP_USER, "idle": f"{IDLE_SESSION_SECONDS}s"}) - if not scram_accepts(app_verifier(), password): - set_app_password(password) + signs_in_with_its_password(APP_USER, password) except Refused as e: return f"the database refused to be set up for the program: {e}" + for role in ROLES: + name = role["name"] + try: + psql(role_script(role), {"role": name, "app": APP_USER}) + if role["login"]: + if stored_password(name) is None: + write_password(name, new_password()) + signs_in_with_its_password(name, stored_password(name)) + except Refused as e: + return f"the database could not set up the role '{name}' the node lists in `roles`: {e}" + except OSError as e: + return f"the password of the role '{name}' the node lists in `roles` could not be written to the database's disk: {e}" + try: + psql(STRIP_UNLISTED, {"app": APP_USER, "admin": ADMIN_USER, "listed": json.dumps([role["name"] for role in ROLES])}) + except Refused as e: + return f"the database could not take back from a role the node no longer lists in `roles` what the list gave it: {e}" for name in EXTENSIONS: try: psql('CREATE EXTENSION IF NOT EXISTS :"name";\n', {"name": name}) @@ -459,9 +584,10 @@ def converge_once() -> str | None: return None -def reset_password() -> str | None: - """Mint a new password, set it on the program's role, tell the pooler, - make it readable again, and tell weft: what went wrong, or None. +def reset_password(role: str) -> str | None: + """Mint a new password for `role` (the program's, or a login role's), set + it on the role, tell the pooler, and for the program's make it readable + again and tell weft: what went wrong, or None. The database is told first, over its unix socket, which trusts a local connection without a password: a file that named a password @@ -474,56 +600,61 @@ def reset_password() -> str | None: # back as it is (Postgres keeps a SCRAM verifier it is handed), not # the old password hashed again under a new salt, which the pooler, # holding this one, could no longer sign in with. - held = app_verifier() + held = verifier(role) if held is None: - return f"the role {APP_USER} has no password in the database, so the reset has nothing to replace; restart the database's infra" + return f"the role {role} has no password in the database, so the reset has nothing to replace; restart the database's infra" try: - set_app_password(password) + set_password(role, password) except Refused as e: - sys.stderr.write(f"credential: reset refused by the database: {e}\n") + sys.stderr.write(f"credential: reset of {role} refused by the database: {e}\n") return f"the database refused the new password: {e}" try: - write_password(password) + write_password(role, password) except OSError as e: # A write that failed after the file was swapped in (its # directory not synced) still left the new password there. try: - written = stored_password() == password + written = stored_password(role) == password except OSError: written = False if not written: - sys.stderr.write(f"credential: reset, but the password could not be written: {e}\n") + sys.stderr.write(f"credential: reset of {role}, but the password could not be written: {e}\n") try: - set_app_password(held) + set_password(role, held) except Refused as undo: return ( - f"the database took a new password that could not be written to its disk ({e}), and would not " - f"take the old one back ({undo}), so no client can sign in. Free some room on the database's " - "disk (or whatever the error says) and press Reset password again, or restart the database's infra." + f"the database took a new password for {role} that could not be written to its disk ({e}), and " + f"would not take the old one back ({undo}), so no client can sign in as {role}. Free some room " + "on the database's disk (or whatever the error says) and press Reset password again, or " + "restart the database's infra." ) return ( f"the new password could not be written to the database's disk ({e}), so the reset was undone and " "the old password still signs in. Free some room on that disk (or whatever the error says) and press Reset password again." ) - sys.stderr.write(f"credential: the new password is written, but its directory could not be synced: {e}\n") + sys.stderr.write(f"credential: the new password of {role} is written, but its directory could not be synced: {e}\n") try: tell_the_pooler() except (Refused, OSError) as e: - sys.stderr.write(f"credential: reset, but the pooler was not told: {e}\n") + sys.stderr.write(f"credential: reset of {role}, but the pooler was not told: {e}\n") return ( - f"the database has the new password, but its pooler could not be told ({e}), so no client can " - "sign in until it is. Press Reset password again." + f"the database has the new password for {role}, but its pooler could not be told ({e}), so no client " + f"can sign in as {role} until it is. Press Reset password again." ) try: - os.remove(SEALED_FILE) + os.remove(sealed_file(role)) except FileNotFoundError: pass except OSError as e: - sys.stderr.write(f"credential: reset, but the password stays sealed: {e}\n") + sys.stderr.write(f"credential: reset of {role}, but the password stays sealed: {e}\n") return ( - f"the database and its pooler have a new password, but it could not be made readable ({e}), so " - "neither the card nor a run can take it. Press Reset password again." + f"the database and its pooler have a new password for {role}, but it could not be made readable ({e}), " + "so nothing can take it. Press Reset password again." ) + if role != APP_USER: + # No connection weft keeps signs in as a login role. + sys.stderr.write(f"credential: password of {role} reset; readable again until it is taken\n") + return None # The connection the node published still holds the old password # until weft is told the new one. unheard = push_password(password) @@ -552,9 +683,9 @@ def push_password(password: str) -> str | None: return str(e) -def stored_password() -> str | None: +def stored_password(role: str) -> str | None: try: - with open(PASSWORD_FILE, encoding="utf-8") as f: + with open(password_file(role), encoding="utf-8") as f: return f.read() except FileNotFoundError: return None @@ -591,7 +722,7 @@ def do_GET(self) -> None: # noqa: N802 (the stdlib's spelling) if self.path != CREDENTIAL_PATH: self._send(404, {"error": "no such path"}) return - password = stored_password() + password = stored_password(APP_USER) if password is None: self._send(503, {"error": "no password on the shared volume yet"}) return @@ -599,7 +730,7 @@ def do_GET(self) -> None: # noqa: N802 (the stdlib's spelling) # that asks needs to act on it (find its own connection, or # tell the user how to recover), and an HTTP error would reach # it as an opaque transport failure instead. - self._send(200, {"sealed": True} if sealed() else {"password": password}) + self._send(200, {"sealed": True} if sealed(APP_USER) else {"password": password}) def _body(self) -> dict: try: @@ -621,31 +752,20 @@ def do_POST(self) -> None: # noqa: N802 if self.path != CREDENTIAL_STORED_PATH: self._send(404, {"error": "no such path"}) return - password = stored_password() - if password is None: - self._send(503, {"error": "no password on the shared volume yet"}) - return - sent = self._body().get("password", "") - # Only a caller that actually holds the password may retire it. - # Compared as BYTES: comparing str refuses non-ASCII outright, - # which would escape as a crash rather than a refusal. - # # A mismatch is an ANSWER, not a failure. It is how the caller # learns that the password it holds is not this database's, # which happens when the disk was replaced under it: a fresh # disk means a fresh password, and the caller has to notice # rather than go on handing out one nothing accepts. - if not secrets.compare_digest(str(sent).encode("utf-8"), password.encode("utf-8")): - self._send(200, {"stored": False}) - return - # Idempotent: confirming twice is the same as confirming once. try: - with open(SEALED_FILE, "w", encoding="utf-8") as f: - f.write("stored") + held = seal_if_held(APP_USER, str(self._body().get("password", ""))) except OSError as e: self._send(503, {"error": f"the password could not be retired, its disk refused the write: {e}"}) return - self._send(200, {"stored": True}) + if held is None: + self._send(503, {"error": "no password on the shared volume yet"}) + return + self._send(200, {"stored": held}) def _action(self, body: dict) -> None: # The graph's button, in the dispatcher's envelope: a refusal @@ -653,13 +773,19 @@ def _action(self, body: dict) -> None: # SYNC: the /action envelope <-> crates/weft-core/src/infra/action.rs, catalog/bailey/bridge/images/bridge/src/actions.js, # catalog/computer/computer/images/computer/computer.py action = body.get("action") - if action != RESET_ACTION: + if (role := role_of(action, TAKEN_ACTION)) is not None: + payload = body.get("payload") + refused = taken(role, payload.get("value") if isinstance(payload, dict) else None) + self._send(200, {"result": {"error": refused} if refused is not None else {"taken": True}}) + return + role = role_of(action, RESET_ACTION) + if role is None: self._send(200, {"result": {"error": f"no such action: {action!r}"}}) return if not CONVERGED.is_set(): self._send(200, {"result": {"error": "the database is still being set up for the program; press it again in a moment"}}) return - refused = reset_password() + refused = reset_password(role) if refused is not None: self._send(200, {"result": {"error": refused}}) return @@ -670,61 +796,108 @@ def log_message(self, fmt: str, *args: object) -> None: sys.stderr.write("credential: %s\n" % (fmt % args)) -def live_items() -> list: - """What the node's card shows: the names to sign in with, the password - itself while it is still readable or the state it has moved on to, with - the one button that makes it readable again, then who is connected. - - The card is a legitimate first taker. The password is readable - exactly once, by whoever asks first, and then sealed; a person - looking at the graph is asking the same way a run does, so while it - is unsealed the card SHOWS it, masked, with a way to copy it. That - is how somebody outside the program (a frontend that needs its own - tables in this database, a `psql` session) gets in without anybody - digging in a container for it. - - Once a run has taken it the value is gone from here for good, and - the way back is the button below: reset, and the fresh password is - readable, and shown, again.""" - password = stored_password() +def role_of(action: object, kind: str) -> str | None: + """The role an `action` of `kind` (a reset, a taken) is for: `kind` alone + names the program's role, `kind:` a login role. None when no + button or press of this card is called that.""" + if action == kind: + return APP_USER + if isinstance(action, str) and action.startswith(kind + ROLE_MARK): + role = action[len(kind + ROLE_MARK):] + return role if role in SIGNS_IN[1:] else None + return None + + +def taken(role: str, value: object) -> str | None: + """A reader copied `role`'s password out of the card: seal it, so the card + shows it handed over until a reset. Only a reader that holds the password + may, as for a run that stored it. What went wrong, or None.""" + try: + held = seal_if_held(role, value) + except OSError as e: + return f"the password of {role} could not be marked handed over, its disk refused the write: {e}" + if held is None: + return f"the role {role} has no password yet, so there is nothing to have taken" + if not held: + return f"what was sent is not the password of {role} any more; read the card again" + return None + + +def seal_if_held(role: str, value: object) -> bool | None: + """Seal `role`'s password when `value` is it: True once sealed, False when + it is not (an answer: the sender holds another password), None when the + role has none yet. Only a sender that holds the password may retire it, + so nothing able to reach this port retires a password nobody holds. + Compared as BYTES: comparing str refuses non-ASCII outright, which would + escape as a crash rather than a refusal. Idempotent. Raises `OSError`.""" + with CHANGING: + password = stored_password(role) + if password is None: + return None + if not isinstance(value, str) or not secrets.compare_digest(value.encode("utf-8"), password.encode("utf-8")): + return False + seal(role) + return True + + +def password_item(role: str) -> dict: + """`role`'s password on the card, one model for every password the node + holds: readable (masked, with reveal and copy) until it is handed over, + to a run that stored it or a reader that copied it out (`taken`); after + that a line saying so; and the button that mints a new one, readable + again, in either state.""" + mark = "" if role == APP_USER else ROLE_MARK + role + label = "Password" if role == APP_USER else f"{role} password" + password = stored_password(role) if password is None: - item = { - "type": "text", - "label": "Password", - "data": "not minted yet", - } - elif sealed(): - item = { - "type": "text", - "label": "Password", - "data": "handed over to a connection; reset to get a new one", - } + item = {"type": "text", "label": label, "data": "not minted yet"} + elif sealed(role): + # SYNC: handedOver <-> crates/weft-core/src/live.rs (LiveItem::handed_over), packages/weft-graph/src/protocol.ts (LiveDataItem.handedOver) + item = {"type": "text", "label": label, "data": "handed over; reset to get a new one", "handedOver": True} else: - # Rendered masked, with reveal and copy beside it. - item = { - "type": "secret", - "label": "Password", - "data": password, - } - # The database and user a client signs in with, beside the - # password. The pooler lets clients into this database only, under - # this name. - names = [ - {"type": "text", "label": "Database", "data": POOLER_DATABASE}, - {"type": "text", "label": "User", "data": APP_USER}, - ] + item = {"type": "secret", "label": label, "data": password, "taken": TAKEN_ACTION + mark} item["action"] = { "label": "Reset password", - "actionKind": RESET_ACTION, + "actionKind": RESET_ACTION + mark, "confirm": ( "Give the database a new password? A client already connected stays " "connected; every new sign-in with the old one is refused. The program's " "own next run picks the new one up by itself; anything outside it (a " "frontend, a psql session) takes it from this card." + ) + if role == APP_USER + else ( + f"Give the role {role} a new password? A client already connected as {role} stays connected; every " + "new sign-in with the old one is refused, so whatever signs in as it takes the new one from this card." ), } + return item + + +def live_items() -> list: + """What the node's card shows: the names to sign in with, then each + password the node holds (the program's, then each login role's), then + who is connected. + + The card is a legitimate first taker. A password is readable exactly + once, by whoever takes it first, and then sealed; a person looking at + the graph is asking the same way a run does, so while it is unsealed + the card SHOWS it, masked, with a way to copy it. That is how somebody + outside the program (a frontend that needs its own tables in this + database, a site signing in as a login role, a `psql` session) gets in + without anybody digging in a container for it. + + Once taken the value is gone from here for good, and the way back is + its Reset button: the fresh password is readable, and shown, again.""" + # The database and user a client signs in with, beside the + # password. The pooler lets clients into this database only, under + # this name. + names = [ + {"type": "text", "label": "Database", "data": POOLER_DATABASE}, + {"type": "text", "label": "User", "data": APP_USER}, + ] warning = [{"type": "text", "label": "Warning", "data": POSTGRES_NOTE}] if POSTGRES_NOTE else [] - return warning + names + [item] + connection_items() + return warning + names + [password_item(role) for role in SIGNS_IN] + connection_items() def connection_items() -> list: @@ -771,15 +944,18 @@ def postgres_connections(limits: dict, activity: list[dict]) -> str: def pooler_lanes(databases: list[dict], pools: list[dict]) -> str: - """The pooler's two doors: clients served and waiting, connections open of its share.""" + """The pooler's two doors: clients served and waiting, connections open of + its share. A door has a pool per role signing in through it, all held to + the door's share together, so its clients are counted across them.""" lines = [] for name, lane in ((POOLER_DATABASE, "queries"), (WATCH_DATABASE, "watchers")): db = next((row for row in databases if row.get("name") == name), {}) - pool = next((row for row in pools if row.get("database") == name and row.get("user") == APP_USER), {}) - served, waiting = int(pool.get("cl_active", 0)), int(pool.get("cl_waiting", 0)) + door = [row for row in pools if row.get("database") == name] + served = sum(int(row.get("cl_active", 0)) for row in door) + waiting = sum(int(row.get("cl_waiting", 0)) for row in door) line = f"{lane} ({name}): {served} client(s) served, {waiting} waiting" if waiting: - line += f" (the longest for {pool.get('maxwait', '0')}s)" + line += f" (the longest for {max(int(row.get('maxwait', 0)) for row in door)}s)" line += f"; {db.get('current_connections', 0)} of {db.get('pool_size', '?')} connections open" lines.append(line) return "\n".join(lines) diff --git a/catalog/postgres/database/images/credential/test_bootstrap.py b/catalog/postgres/database/images/credential/test_bootstrap.py index 26de19196..7ffd5bcd3 100644 --- a/catalog/postgres/database/images/credential/test_bootstrap.py +++ b/catalog/postgres/database/images/credential/test_bootstrap.py @@ -2,8 +2,10 @@ for the pooler, and what the card shows. Run with the standard library alone (`python3 -m unittest`), so it needs no install.""" +import json import os import sys +import tempfile import unittest # The settings the node hands the container, so the module imports here @@ -12,7 +14,11 @@ "WEFT_SECRET_DIR": "/store/secret", "WEFT_PASSWORD_FILE": "/store/secret/password", "WEFT_CREDENTIAL_PORT": "8099", "WEFT_SOCKET_DIR": "/var/run/postgresql", "WEFT_POSTGRES_PORT": "5433", "WEFT_ADMIN_USER": "weft", "WEFT_APP_USER": "app", "WEFT_DATABASE": "my shop", "WEFT_POOLER_DATABASE": "shop", "WEFT_WATCH_DATABASE": "shop_watch", - "WEFT_EXTENSIONS": '["pg_trgm", "nope"]', "WEFT_POSTGRES_MAJOR": "17", "WEFT_CLOSES_IDLE_SESSIONS_FROM": "14", + "WEFT_EXTENSIONS": '["pg_trgm", "nope"]', + "WEFT_ROLES": '[{"name": "tenant_app", "login": false, "bypassRowLevelSecurity": false},' + ' {"name": "tenant_site", "login": true, "bypassRowLevelSecurity": false},' + ' {"name": "tenant_admin", "login": false, "bypassRowLevelSecurity": true}]', + "WEFT_POSTGRES_MAJOR": "17", "WEFT_CLOSES_IDLE_SESSIONS_FROM": "14", "WEFT_POSTGRES_NOTE": "", "WEFT_DATA_DIR": "/store/pgdata", "WEFT_IDLE_SESSION_SECONDS": "60", "WEFT_POOLER_DIR": "/run/pooler", "WEFT_POOLER_PORT": "5432", "WEFT_POOLER_USERS_FILE": "/run/pooler/users", "WEFT_VALUES_URL": "http://agent/values", @@ -39,7 +45,90 @@ def test_no_verifier_or_another_kind_or_a_broken_one_is_a_no(self): class PoolerUsersTest(unittest.TestCase): def test_the_program_signs_in_with_its_verifier_and_the_superuser_with_none(self): - self.assertEqual(bootstrap.pooler_users(STORED), f'"app" "{STORED}"\n"weft" ""\n') + self.assertEqual(bootstrap.pooler_users({"app": STORED}), f'"app" "{STORED}"\n"weft" ""\n') + + def test_each_login_role_signs_in_with_its_own_verifier(self): + self.assertEqual( + bootstrap.pooler_users({"app": STORED, "tenant_site": "SCRAM-other"}), + f'"app" "{STORED}"\n"tenant_site" "SCRAM-other"\n"weft" ""\n', + ) + + def test_only_the_program_and_the_login_roles_sign_in(self): + self.assertEqual(bootstrap.SIGNS_IN, ["app", "tenant_site"]) + + +class RolesTest(unittest.TestCase): + def test_a_role_is_made_with_the_words_its_entry_picks_and_granted_to_the_program(self): + site = bootstrap.role_script({"name": "tenant_site", "login": True, "bypassRowLevelSecurity": False}) + self.assertIn("NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION LOGIN NOBYPASSRLS;", site) + self.assertIn('GRANT :"role" TO :"app";', site) + self.assertNotIn("tenant_site", site, "the name travels as a psql variable, never in the SQL") + admin = bootstrap.role_script({"name": "tenant_admin", "login": False, "bypassRowLevelSecurity": True}) + self.assertIn(" NOLOGIN BYPASSRLS;", admin) + + def test_the_node_s_roles_are_read_whole_or_refused(self): + self.assertEqual(bootstrap.read_roles("[]"), []) + for broken in ('{"name": "x"}', '[{"name": "x"}]', '[{"name": "", "login": false, "bypassRowLevelSecurity": false}]', + '[{"name": "x", "login": "yes", "bypassRowLevelSecurity": false}]'): + with self.assertRaises(SystemExit, msg=broken): + bootstrap.read_roles(broken) + + def test_each_button_and_each_taken_press_names_its_own_role(self): + for kind in ("reset_password", "password_taken"): + self.assertEqual(bootstrap.role_of(kind, kind), "app") + self.assertEqual(bootstrap.role_of(f"{kind}:tenant_site", kind), "tenant_site") + for nothing in (f"{kind}:tenant_app", f"{kind}:weft", f"{kind}:", "other", None, 3): + self.assertIsNone(bootstrap.role_of(nothing, kind), nothing) + self.assertIsNone(bootstrap.role_of("password_taken", "reset_password")) + + +class PasswordCardTest(unittest.TestCase): + """Every password the node holds is shown the one way: readable until it + is taken, then a line saying so, with its Reset button in both states.""" + + def setUp(self): + self.kept = {name: getattr(bootstrap, name) for name in ("stored_password", "sealed", "seal")} + self.passwords = {"app": "pw", "tenant_site": "site-pw"} + self.sealed = set() + bootstrap.stored_password = lambda role: self.passwords.get(role) + bootstrap.sealed = lambda role: role in self.sealed + bootstrap.seal = lambda role: self.sealed.add(role) + + def tearDown(self): + for name, value in self.kept.items(): + setattr(bootstrap, name, value) + + def test_a_password_is_readable_with_its_taken_press_until_taken(self): + for role, label, mark in (("app", "Password", ""), ("tenant_site", "tenant_site password", ":tenant_site")): + item = bootstrap.password_item(role) + self.assertEqual((item["type"], item["label"], item["data"]), ("secret", label, self.passwords[role])) + self.assertEqual(item["taken"], "password_taken" + mark) + self.assertEqual(item["action"]["actionKind"], "reset_password" + mark) + + def test_only_the_password_itself_retires_it(self): + self.assertIn("not the password", bootstrap.taken("tenant_site", "wrong")) + self.assertIn("not the password", bootstrap.taken("tenant_site", None)) + self.assertEqual(self.sealed, set()) + self.assertIsNone(bootstrap.taken("tenant_site", "site-pw")) + self.assertEqual(self.sealed, {"tenant_site"}, "the role taken, never the program's") + item = bootstrap.password_item("tenant_site") + self.assertEqual((item["type"], item["data"], item["handedOver"]), ("text", "handed over; reset to get a new one", True)) + self.assertNotIn("taken", item) + self.assertEqual(item["action"]["actionKind"], "reset_password:tenant_site") + + def test_a_password_not_minted_yet_cannot_be_taken(self): + self.passwords.pop("tenant_site") + self.assertEqual(bootstrap.password_item("tenant_site")["data"], "not minted yet") + self.assertIn("no password yet", bootstrap.taken("tenant_site", "x")) + + def test_the_card_holds_every_password_in_order(self): + kept = bootstrap.connection_items + bootstrap.connection_items = lambda: [] + try: + labels = [item["label"] for item in bootstrap.live_items()] + self.assertEqual(labels, ["Database", "User", "Password", "tenant_site password"]) + finally: + bootstrap.connection_items = kept class RowsTest(unittest.TestCase): @@ -79,6 +168,15 @@ def test_the_pooler_shows_each_door_with_who_waits(self): "watchers (shop_watch): 1 client(s) served, 0 waiting; 1 of 4 connections open", ) + def test_a_door_counts_the_clients_of_every_role_signing_in_through_it(self): + databases = [{"name": "shop", "pool_size": "15", "current_connections": "15"}] + pools = [ + {"database": "shop", "user": "app", "cl_active": "10", "cl_waiting": "2", "maxwait": "1"}, + {"database": "shop", "user": "tenant_site", "cl_active": "5", "cl_waiting": "3", "maxwait": "4"}, + ] + shown = bootstrap.pooler_lanes(databases, pools) + self.assertTrue(shown.startswith("queries (shop): 15 client(s) served, 5 waiting (the longest for 4s)"), shown) + def test_a_door_nobody_used_yet_reads_as_empty(self): shown = bootstrap.pooler_lanes([{"name": "shop", "pool_size": "15", "current_connections": "0"}], []) self.assertTrue(shown.startswith("queries (shop): 0 client(s) served, 0 waiting; 0 of 15 connections open"), shown) @@ -101,7 +199,8 @@ class ConvergeTest(unittest.TestCase): def setUp(self): self.sent = [] - self.kept = {name: getattr(bootstrap, name) for name in ("psql", "wait_until", "stored_password", "app_verifier")} + self.kept = {name: getattr(bootstrap, name) for name in ("psql", "wait_until", "stored_password", "verifier", "write_password")} + self.passwords = {"app": "pw"} def psql(script, variables, database=None, pooler=False): self.sent.append((script, dict(variables))) @@ -111,8 +210,9 @@ def psql(script, variables, database=None, pooler=False): bootstrap.psql = psql bootstrap.wait_until = lambda what, not_yet: None - bootstrap.stored_password = lambda: "pw" - bootstrap.app_verifier = lambda: STORED + bootstrap.stored_password = lambda role: self.passwords.get(role) + bootstrap.write_password = lambda role, password: self.passwords.__setitem__(role, password) + bootstrap.verifier = lambda role: STORED def tearDown(self): for name, value in self.kept.items(): @@ -126,6 +226,37 @@ def test_each_listed_extension_is_made_and_one_postgres_lacks_is_refused_by_name self.assertIn("'nope'", refused) self.assertIn("is not available", refused) + def test_each_role_is_made_and_a_login_role_gets_a_password_of_its_own_once(self): + bootstrap.converge_once() + made = [variables["role"] for script, variables in self.sent if "CREATE ROLE" in script and "role" in variables] + self.assertEqual(made, ["tenant_app", "tenant_site", "tenant_admin"]) + minted = self.passwords["tenant_site"] + self.assertNotEqual(minted, "pw") + self.assertNotIn("tenant_app", self.passwords, "a role that does not sign in has no password") + given = [variables["password"] for script, variables in self.sent if "PASSWORD" in script and variables.get("role") == "tenant_site"] + self.assertEqual(given, [minted], "the role is given the password just minted, which the verifier did not accept") + self.sent.clear() + bootstrap.converge_once() + self.assertEqual(self.passwords["tenant_site"], minted, "kept from then on") + + def test_the_roles_listed_no_more_lose_what_the_list_gave_them(self): + bootstrap.converge_once() + [(script, variables)] = [(script, variables) for script, variables in self.sent if "REVOKE" in script] + self.assertEqual(json.loads(variables["listed"]), ["tenant_app", "tenant_site", "tenant_admin"]) + self.assertIn("NOLOGIN NOBYPASSRLS", script) + self.assertIn("m.grantor", script, "only what the superuser granted, never a grant somebody else made") + + def test_a_role_the_database_refuses_is_named(self): + def psql(script, variables, database=None, pooler=False): + if variables.get("role") == "tenant_site" and "CREATE ROLE" in script: + raise bootstrap.Refused("ERROR: role \"app\" is a member of role \"tenant_site\"") + return "" + + bootstrap.psql = psql + refused = bootstrap.converge_once() + self.assertIn("'tenant_site'", refused) + self.assertIn("`roles`", refused) + def test_a_pass_the_database_takes_answers_nothing(self): bootstrap.EXTENSIONS, kept = ["pg_trgm"], bootstrap.EXTENSIONS try: @@ -143,18 +274,19 @@ def test_a_new_database_is_never_made_on_a_postgres_that_keeps_idle_sessions(sel def test_the_idle_limit_is_set_only_where_postgres_has_it(self): sent = [] - kept = {name: getattr(bootstrap, name) for name in ("psql", "wait_until", "stored_password", "app_verifier", "CLOSES_IDLE_SESSIONS", "EXTENSIONS")} - bootstrap.psql = lambda script, variables, database=None, pooler=False: sent.append(script) or "" + kept = {name: getattr(bootstrap, name) for name in ("psql", "wait_until", "stored_password", "verifier", "CLOSES_IDLE_SESSIONS", "EXTENSIONS")} + bootstrap.psql = lambda script, variables, database=None, pooler=False: sent.append((script, variables)) or "" bootstrap.wait_until = lambda what, not_yet: None - bootstrap.stored_password = lambda: "pw" - bootstrap.app_verifier = lambda: STORED + bootstrap.stored_password = lambda role: "pw" + bootstrap.verifier = lambda role: STORED bootstrap.EXTENSIONS = [] try: for closes, expected in ((True, True), (False, False)): sent.clear() bootstrap.CLOSES_IDLE_SESSIONS = closes self.assertIsNone(bootstrap.converge_once()) - self.assertEqual(any("idle_session_timeout" in script for script in sent), expected) + closed = [variables["role"] for script, variables in sent if "idle_session_timeout" in script] + self.assertEqual(closed, ["app", "tenant_site"] if expected else [], "every role that signs in") finally: for name, value in kept.items(): setattr(bootstrap, name, value) @@ -163,7 +295,7 @@ def test_the_card_warns_of_an_old_postgres_first(self): kept = {name: getattr(bootstrap, name) for name in ("POSTGRES_NOTE", "connection_items", "stored_password")} bootstrap.POSTGRES_NOTE = "Postgres 13 does not close a connection a client left idle" bootstrap.connection_items = lambda: [] - bootstrap.stored_password = lambda: None + bootstrap.stored_password = lambda role: None try: items = bootstrap.live_items() self.assertEqual(items[0], {"type": "text", "label": "Warning", "data": bootstrap.POSTGRES_NOTE}) @@ -179,14 +311,14 @@ class ResetUndoTest(unittest.TestCase): exact verifier it held, which the pooler still signs in with.""" def setUp(self): - self.names = ("app_verifier", "set_app_password", "write_password", "stored_password", "tell_the_pooler", "push_password") + self.names = ("verifier", "set_password", "write_password", "stored_password", "tell_the_pooler", "push_password") self.kept = {name: getattr(bootstrap, name) for name in self.names} self.set = [] - bootstrap.app_verifier = lambda: STORED - bootstrap.set_app_password = lambda password: self.set.append(password) - bootstrap.stored_password = lambda: "pw" + bootstrap.verifier = lambda role: STORED + bootstrap.set_password = lambda role, password: self.set.append((role, password)) + bootstrap.stored_password = lambda role: "pw" - def full(password): + def full(role, password): raise OSError(28, "No space left on device") bootstrap.write_password = full @@ -198,10 +330,37 @@ def tearDown(self): setattr(bootstrap, name, value) def test_the_old_verifier_comes_back_as_it_was(self): - refused = bootstrap.reset_password() - self.assertEqual(len(self.set), 2) - self.assertEqual(self.set[1], STORED, "the verifier itself, never the password hashed again") - self.assertIn("undone", refused) + for role in ("app", "tenant_site"): + self.set.clear() + refused = bootstrap.reset_password(role) + self.assertEqual(len(self.set), 2) + self.assertEqual(self.set[1], (role, STORED), "the verifier itself, never the password hashed again") + self.assertIn("undone", refused) + + +class RoleResetTest(unittest.TestCase): + """A login role's reset tells the pooler and makes its password readable + again, and stops there: weft holds no connection for it.""" + + def test_a_login_role_s_reset_unseals_it_and_tells_no_connection(self): + names = ("verifier", "set_password", "write_password", "tell_the_pooler", "push_password", "sealed_file") + kept = {name: getattr(bootstrap, name) for name in names} + told = [] + bootstrap.verifier = lambda role: STORED + bootstrap.set_password = lambda role, password: None + bootstrap.write_password = lambda role, password: None + bootstrap.tell_the_pooler = lambda: told.append("pooler") + bootstrap.push_password = lambda password: self.fail("weft holds no connection for a login role") + seal = os.path.join(tempfile.mkdtemp(), "sealed") + open(seal, "w").close() + bootstrap.sealed_file = lambda role: seal + try: + self.assertIsNone(bootstrap.reset_password("tenant_site")) + self.assertEqual(told, ["pooler"]) + self.assertFalse(os.path.exists(seal), "readable again") + finally: + for name, value in kept.items(): + setattr(bootstrap, name, value) if __name__ == "__main__": diff --git a/catalog/postgres/database/metadata.json b/catalog/postgres/database/metadata.json index d11b28976..2b43a8726 100644 --- a/catalog/postgres/database/metadata.json +++ b/catalog/postgres/database/metadata.json @@ -1,7 +1,7 @@ { "type": "PostgresDatabase", "label": "Postgres: Database", - "description": "Run a Postgres database inside the project, with a disk that survives restarts; wire its access output into the query and write nodes. Its generated password is handed out exactly ONCE, to whoever asks first: until then the node's card shows it masked with a copy button (how an outside client, such as a frontend with its own tables, gets in); afterwards the card's Reset password button mints and shows a new one. `weft infra env --into .env --set DATABASE_USER=User --set DATABASE_PASSWORD=Password` writes both into an env file without anyone reading the password; once handed over, `weft infra press reset_password` then `weft infra env` writes a fresh one (`weft infra show ` lists the buttons). Every client (the program's workers, a frontend) signs in as `app`, which owns the database and is no superuser, through the node's pooler: Postgres serves a fixed number of connections (see `memory`), and a client past them waits its turn instead of being refused, however many workers run. A connection that sits idle for a minute is closed by the database, so an outside client's own pool must expect an idle connection to be gone (most close theirs sooner). Watching queries (PostgresWatchQuery) have their own share of the connections, a quarter of them. Through the pooler a client reaches this one database only, never Postgres's own `postgres` database or another one made beside it. An extension the program needs, trusted by Postgres or not, goes in `extensions`: the superuser `weft` makes it on every start, and `app` uses it. The superuser signs in only over the unix socket inside the database's own container. Postgres note: a function called in a `cross join lateral (select f(x) as v)` subquery and read several times through `v` can run once per read, because Postgres folds the subquery into the outer query; `offset 0` inside the subquery keeps it to one call. Terminating its infra deletes the disk and all data.", + "description": "Run a Postgres database inside the project, with a disk that survives restarts; wire its access output into the query and write nodes. Its generated password is handed out exactly ONCE, to whoever takes it first (the program's run, `weft infra env`, or the card's copy button): until then the node's card shows it masked with a copy button (how an outside client, such as a frontend with its own tables, gets in); afterwards the card's Reset password button mints and shows a new one. `weft infra env --into .env --set DATABASE_USER=User --set DATABASE_PASSWORD=Password` writes both into an env file without anyone reading the password; once handed over, `weft infra press reset_password` then `weft infra env` writes a fresh one (`weft infra show ` lists the buttons). Every client (the program's workers, a frontend) signs in as `app`, which owns the database and is no superuser, through the node's pooler: Postgres serves a fixed number of connections (see `memory`), and a client past them waits its turn instead of being refused, however many workers run. A connection that sits idle for a minute is closed by the database, so an outside client's own pool must expect an idle connection to be gone (most close theirs sooner). Watching queries (PostgresWatchQuery) have their own share of the connections, a quarter of them. Through the pooler a client reaches this one database only, never Postgres's own `postgres` database or another one made beside it. An extension the program needs, trusted by Postgres or not, goes in `extensions`: the superuser `weft` makes it on every start, and `app` uses it. Roles the program needs go in `roles`, made the same way. To keep tenants apart with row-level security: list a role per kind of tenant, have the program grant it what it may touch, turn on (and `FORCE`) the tables' policies naming it, and `set role` to it before a tenant's queries; a client outside the program (a site) signs in as a `login` role of its own. `app` owns the tables, and a `FORCE` policy binds their owner too, so a setup step that must write past the policies (seeding every tenant's rows) does `set role` to a role marked `bypassRowLevelSecurity`, granted what it writes. The superuser signs in only over the unix socket inside the database's own container. Postgres note: a function called in a `cross join lateral (select f(x) as v)` subquery and read several times through `v` can run once per read, because Postgres folds the subquery into the outer query; `offset 0` inside the subquery keeps it to one call. Terminating its infra deletes the disk and all data.", "tags": [ "postgres", "database", @@ -48,6 +48,13 @@ "label": "Extensions", "description": "Extensions the program's database needs, as Postgres names them (`[\"pg_trgm\", \"postgres_fdw\"]`). On every start the superuser makes each one that is not there yet, so the program, signed in as `app`, uses them, including those Postgres does not let a non-superuser make. One this Postgres does not have fails the start, and `weft infra status` names it. Taking one off the list leaves it in the database. Changing the list restarts the database." }, + { + "name": "roles", + "type": "List[{ name: String, login?: Boolean, bypassRowLevelSecurity?: Boolean }]", + "default": [], + "label": "Roles", + "description": "Postgres roles the program needs beside `app`, such as one per kind of tenant that row-level security policies name (`[{\"name\": \"tenant_app\"}, {\"name\": \"tenant_site\", \"login\": true}, {\"name\": \"tenant_admin\", \"bypassRowLevelSecurity\": true}]`). A name is lowercase letters, digits and `_`, the way a program writes it in `set role`. On every start the superuser makes each one that is missing and sets what the entry says (none is ever a superuser or makes roles), and grants it to `app`, so the program can `set role tenant_app` and grant it what it needs. `login` (off by default): the role signs in by itself through the pooler, with its own password, handed out once like `app`'s: the card shows it under ` password` until it is taken (`weft infra env --into .env --set SITE_PASSWORD=\"tenant_site password\"` writes it, or the card's copy button), and its own Reset button mints a new one; `app`'s password does not sign it in. `bypassRowLevelSecurity` (off by default): the role reads and writes past every policy, `FORCE` ones included. Taking a role off the list takes back what the list gave it: `app` can no longer `set role` to it, it signs in no more (its card line and its pooler lines go) and it passes row-level security policies no more; the role stays in the database with what it owns. Changing the list restarts the database." + }, { "name": "reachable", "type": "Boolean", @@ -140,7 +147,7 @@ "regex": "^([1-9]|1[0-3])(\\.(0|[1-9][0-9]*))?$" }, "then": { - "message": "PostgresDatabase '{id}' runs a Postgres older than 14, which does not close connections a client left idle: an existing database keeps running without that, and a new one is refused when it starts. To move to 14 or later: Postgres cannot open an older version's data, so the data moves by dump and restore while the old database still runs. Dump it: `weft infra press {id} reset_password`, then `weft infra env {id} --into old.env --set PGUSER=User --set PGDATABASE=Database --as PGPASSWORD`, and with those exported, `pg_dump -Fc -h -p -f data.dump` at the address `weft infra list-doors` prints for it (turn `reachable` on first if it is off). Restore it: add a PostgresDatabase on 17 with `reachable` on, start it, write its env the same way, `pg_restore --no-owner -h -p -d \"$PGDATABASE\" data.dump`, and wire the new node where the old one was.", + "message": "PostgresDatabase '{id}' runs a Postgres older than 14, which does not close connections a client left idle: an existing database keeps running without that, and a new one is refused when it starts. To move to 14 or later: Postgres cannot open an older version's data, so the data moves by dump and restore while the old database still runs. Dump it: `weft infra press {id} reset_password`, then `weft infra env {id} --into old.env --set PGUSER=User --set PGDATABASE=Database --set PGPASSWORD=Password`, and with those exported, `pg_dump -Fc -h -p -f data.dump` at the address `weft infra list-doors` prints for it (turn `reachable` on first if it is off). Restore it: add a PostgresDatabase on 17 with `reachable` on, wire it where the old one was, start it (its setup takes the password), then press its reset_password and write its env the same way, and `pg_restore --no-owner -h -p -d \"$PGDATABASE\" data.dump`.", "level": "structural", "severity": "warning", "port": "version" diff --git a/catalog/postgres/database/mod.rs b/catalog/postgres/database/mod.rs index c94d99fdc..499a07d36 100644 --- a/catalog/postgres/database/mod.rs +++ b/catalog/postgres/database/mod.rs @@ -170,11 +170,11 @@ fn secret_mount() -> Mount { /// paths, the ports and the names are declared once, here, and the /// container cannot drift from them. The pooler's name for the database /// and the program's user are what its card shows beside the password: -/// the three a client signs in with. The extensions are the ones it makes -/// on every start, as JSON, since a container's settings hold no line -/// break. +/// the three a client signs in with. The extensions and the roles are the +/// ones it makes on every start, as JSON, since a container's settings hold +/// no line break. // SYNC: credential_env <-> catalog/postgres/database/images/credential/bootstrap.py (the container's settings) -fn credential_env(database: &str, extensions: &[String], version: &PostgresVersion) -> Vec { +fn credential_env(database: &str, extensions: &[String], roles: &[Role], version: &PostgresVersion) -> Vec { vec![ EnvEntry::new("WEFT_POSTGRES_MAJOR", version.major.to_string()), EnvEntry::new("WEFT_CLOSES_IDLE_SESSIONS_FROM", PostgresVersion::CLOSES_IDLE_SESSIONS.to_string()), @@ -185,6 +185,7 @@ fn credential_env(database: &str, extensions: &[String], version: &PostgresVersi EnvEntry::new("WEFT_POOLER_DATABASE", pooler_database(database)), EnvEntry::new("WEFT_WATCH_DATABASE", watch_database(database)), EnvEntry::new("WEFT_EXTENSIONS", serde_json::to_string(extensions).expect("a list of names serializes")), + EnvEntry::new("WEFT_ROLES", serde_json::to_string(roles).expect("a list of roles serializes")), EnvEntry::new("WEFT_SECRET_DIR", SECRET_PATH), EnvEntry::new("WEFT_PASSWORD_FILE", password_file()), EnvEntry::new("WEFT_CREDENTIAL_PORT", CREDENTIAL_PORT.to_string()), @@ -219,11 +220,11 @@ fn pooler_env(tuning: &Tuning) -> Vec { /// one line each: `--config` and the lines of [`pooler_config`], then /// `--hba` and the lines of [`pooler_hba`]. Arguments rather than /// environment, which cannot carry a line break. -pub fn pooler_args(database: &str, tuning: &Tuning) -> Vec { +pub fn pooler_args(database: &str, tuning: &Tuning, roles: &[Role]) -> Vec { std::iter::once("--config".to_string()) .chain(pooler_config(database, tuning)) .chain(std::iter::once("--hba".to_string())) - .chain(pooler_hba(database)) + .chain(pooler_hba(database, roles)) .collect() } @@ -246,6 +247,15 @@ pub fn watch_database(database: &str) -> String { /// database the way it waits for anything else, and stops when the run /// does. /// +/// PgBouncer keeps a pool per user of a door, each up to `pool_size`, so +/// with a login role of the node's own ([`Role::login`]) beside `app` +/// the pools of a door together could open more than its share; +/// `max_db_connections` holds them all to that share, whoever signs in. +/// A client of one role never waits on another role's idle connections: +/// with the share full, PgBouncer (1.25, measured) closes an idle +/// connection of another pool to serve it (its log says `evicted`), so +/// only connections clients are holding make a client wait. +/// /// No password is in it: who may sign in is the file the credential /// container writes from the database itself. pub fn pooler_config(database: &str, tuning: &Tuning) -> Vec { @@ -253,10 +263,11 @@ pub fn pooler_config(database: &str, tuning: &Tuning) -> Vec { // The database's own name, quoted the way PgBouncer reads a value // (a quote doubled), so any name Postgres has reaches it. let backend = format!("host=127.0.0.1 port={POSTGRES_PORT} dbname='{}'", database.replace('\'', "''")); + let share = |pool: u64| format!("pool_size={pool} max_db_connections={pool}"); [ "[databases]".to_string(), - format!("{door} = {backend} pool_size={}", tuning.query_pool), - format!("{watch} = {backend} pool_size={}", tuning.watch_pool), + format!("{door} = {backend} {}", share(tuning.query_pool)), + format!("{watch} = {backend} {}", share(tuning.watch_pool)), String::new(), "[pgbouncer]".to_string(), "listen_addr = *".to_string(), @@ -290,20 +301,21 @@ pub fn pooler_config(database: &str, tuning: &Tuning) -> Vec { .into() } -/// Who may sign in to the pooler, and how. The program's role, with its -/// password, to the program's database and its watching door, from -/// anywhere the endpoint lets in; the superuser only to the pooler's -/// admin console, only over the pooler's unix socket, which only the -/// credential container mounts. Nobody reaches the superuser through -/// the pooler. -pub fn pooler_hba(database: &str) -> Vec { +/// Who may sign in to the pooler, and how. The program's role and each +/// login role the node lists, each with its own password, to the +/// program's database and its watching door, from anywhere the endpoint +/// lets in; the superuser only to the pooler's admin console, only over +/// the pooler's unix socket, which only the credential container mounts. +/// Nobody reaches the superuser through the pooler, and a role taken off +/// the list loses its lines here, so the pooler lets it in no more. +pub fn pooler_hba(database: &str, roles: &[Role]) -> Vec { let (door, watch) = (pooler_database(database), watch_database(database)); - [ - format!("local pgbouncer {ADMIN_USER} trust"), - format!("host {door},{watch} {APP_USER} 0.0.0.0/0 scram-sha-256"), - format!("host {door},{watch} {APP_USER} ::/0 scram-sha-256"), - ] - .into() + let signs_in = std::iter::once(APP_USER).chain(roles.iter().filter(|role| role.login).map(|role| role.name.as_str())); + std::iter::once(format!("local pgbouncer {ADMIN_USER} trust")) + .chain(signs_in.flat_map(|user| { + ["0.0.0.0/0", "::/0"].map(|from| format!("host {door},{watch} {user} {from} scram-sha-256")) + })) + .collect() } /// The file of who may sign in to the pooler, written by the credential @@ -562,17 +574,96 @@ pub fn extension_names(raw: Vec) -> WeftResult> { Ok(raw) } +/// One entry of the `roles` input: a Postgres role the program's database +/// needs beside `app`, made by the superuser on every start +/// (`bootstrap.py`, `converge`), never a superuser and never one that +/// makes roles, and granted to `app` so the program can `set role` to it. +// SYNC: Role <-> catalog/postgres/database/images/credential/bootstrap.py (WEFT_ROLES), catalog/postgres/database/metadata.json (the `roles` input) +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Role { + pub name: String, + /// Signs in by itself, through the pooler, with a password of its own + /// the card shows: how a client outside the program (a site) signs in + /// as a role other than `app`. + #[serde(default)] + pub login: bool, + /// Reads and writes past every row-level security policy, `FORCE` + /// ones included. + #[serde(default)] + pub bypass_row_level_security: bool, +} + +/// The longest role name Postgres keeps whole. +const MOST_ROLE_CHARS: usize = 63; + +/// The words Postgres reserves (`pg_get_keywords()`, category `R`, as of +/// Postgres 17): written unquoted, `set role user` or `create policy ... to +/// select` is a syntax error, so a role named one of them is one no program +/// can name the way the node tells it to. +const RESERVED_WORDS: [&str; 78] = [ + "all", "analyse", "analyze", "and", "any", "array", "as", "asc", "asymmetric", "both", "case", "cast", "check", + "collate", "column", "constraint", "create", "current_catalog", "current_date", "current_role", "current_time", + "current_timestamp", "current_user", "default", "deferrable", "desc", "distinct", "do", "else", "end", "except", + "false", "fetch", "for", "foreign", "from", "grant", "group", "having", "in", "initially", "intersect", "into", + "lateral", "leading", "limit", "localtime", "localtimestamp", "not", "null", "offset", "on", "only", "or", + "order", "placing", "primary", "references", "returning", "select", "session_user", "some", "symmetric", + "system_user", "table", "then", "to", "trailing", "true", "union", "unique", "user", "using", "variadic", "when", + "where", "window", "with", +]; + +/// The `roles` input, each role checked. A name is what a program writes +/// unquoted in `set role` and a policy (lowercase letters, digits and `_`, +/// not starting with a digit, at most 63 characters, not a word Postgres +/// reserves), so the name it writes is the role it gets; the pooler's +/// sign-in rules take it as written too. Refused: `app` and `weft` (the +/// program's role and the superuser, which this node makes itself), the +/// names Postgres keeps (`public`, `none`, `pg_`, and its reserved words, +/// among them `all`, every user to the pooler, and the ones that mean the +/// current user), and a name listed twice. +pub fn role_list(raw: Vec) -> WeftResult> { + const KEPT: [&str; 4] = [APP_USER, ADMIN_USER, "public", "none"]; + for (i, role) in raw.iter().enumerate() { + let name = role.name.as_str(); + let plain = name.chars().next().is_some_and(|c| c.is_ascii_lowercase() || c == '_') + && name.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') + && name.len() <= MOST_ROLE_CHARS; + if !plain { + mistake!( + "role {name:?} in `roles` is not a name a program can write as it is; use lowercase letters, digits \ + and `_`, not starting with a digit, at most {MOST_ROLE_CHARS} characters (tenant_app)" + ); + } + if KEPT.contains(&name) || name.starts_with("pg_") { + mistake!( + "role {name:?} in `roles` is a name this node or Postgres keeps for itself (`{APP_USER}` is the \ + program's own role, `{ADMIN_USER}` the superuser, `pg_` names and `public` and `none` are \ + Postgres's); name it something else" + ); + } + if RESERVED_WORDS.contains(&name) { + mistake!( + "role {name:?} in `roles` is a word Postgres reserves, so `set role {name}` written as it is would be \ + a syntax error; name it something else (tenant_{name})" + ); + } + if raw[..i].iter().any(|other| other.name == name) { + mistake!("role {name:?} is listed twice in `roles`; list each role once, with all its settings"); + } + } + Ok(raw) +} + /// How a database on a Postgres older than 14 moves to 14 or later: dump /// and restore, while it still runs (its card answers, and when it was set /// up with `reachable` on its address is a door on this machine). // SYNC: MOVE_TO_14 <-> catalog/postgres/database/metadata.json (the old `version` rule's message) const MOVE_TO_14: &str = "Postgres cannot open an older version's data, so the data moves by dump and restore while \ the old database still runs. Dump it: `weft infra press reset_password`, then `weft infra env \ - --into old.env --set PGUSER=User --set PGDATABASE=Database --as PGPASSWORD`, and with those exported, `pg_dump \ + --into old.env --set PGUSER=User --set PGDATABASE=Database --set PGPASSWORD=Password`, and with those exported, `pg_dump \ -Fc -h -p -f data.dump` at the address `weft infra list-doors` prints for it (turn `reachable` on \ - first if it is off). Restore it: add a PostgresDatabase on 17 with `reachable` on, start it, write its env the \ - same way, `pg_restore --no-owner -h -p -d \"$PGDATABASE\" data.dump`, and wire the new node where \ - the old one was"; + first if it is off). Restore it: add a PostgresDatabase on 17 with `reachable` on, wire it where the old one was, \ + start it (its setup takes the password), then press its reset_password and write its env the same way, and `pg_restore --no-owner -h -p -d \"$PGDATABASE\" data.dump`"; /// The Postgres a `version` input names: the image tag, as written, and /// its major version. @@ -657,6 +748,7 @@ impl Node for PostgresDatabaseNode { let machine = MachineShape { cpu: input.opt("cpu")?, memory: Some(memory.clone()), kind: input.opt::("machineType")?.filter(|t| !t.trim().is_empty()), gpu: None }; let tuning = Tuning::for_machine(&memory)?; let extensions = extension_names(input.opt::>("extensions")?.unwrap_or_default())?; + let roles = role_list(input.opt::>("roles")?.unwrap_or_default())?; let credential = Image::Local { name: "credential".into() }; Ok(InfraSpec { @@ -667,7 +759,7 @@ impl Node for PostgresDatabaseNode { name: "db".into(), init_containers: vec![Container::new("mint", credential.clone()) .with_args(vec!["mint".into()]) - .with_env(credential_env(&database, &extensions, &version)) + .with_env(credential_env(&database, &extensions, &roles, &version)) .with_mounts(vec![store_mount()])], containers: vec![ Container::new("postgres", Image::Upstream { @@ -728,7 +820,7 @@ impl Node for PostgresDatabaseNode { // on this one, which knows why (`/health`). Container::new("credential", credential) .with_args(vec!["serve".into()]) - .with_env(credential_env(&database, &extensions, &version)) + .with_env(credential_env(&database, &extensions, &roles, &version)) .with_ports(vec![ContainerPort { name: "http".into(), port: CREDENTIAL_PORT, @@ -740,7 +832,7 @@ impl Node for PostgresDatabaseNode { Probe::http(HEALTH_PATH, CREDENTIAL_PORT).with_initial_delay(2), ), Container::new("pooler", Image::Local { name: "pooler".into() }) - .with_args(pooler_args(&database, &tuning)) + .with_args(pooler_args(&database, &tuning, &roles)) .with_env(pooler_env(&tuning)) .with_ports(vec![ContainerPort { name: "sql".into(), diff --git a/catalog/postgres/database/tests.rs b/catalog/postgres/database/tests.rs index 2d064f6a1..87c4eee3d 100644 --- a/catalog/postgres/database/tests.rs +++ b/catalog/postgres/database/tests.rs @@ -7,8 +7,8 @@ use serde_json::json; use weft::{EndpointMethod, FakeRig, NodeTest, WeftResult}; use super::{ - database_name, extension_names, memory_mib, PostgresVersion, pooler_config, pooler_database, pooler_hba, postgres_version, watch_database, - PostgresDatabaseNode, Tuning, + database_name, extension_names, memory_mib, PostgresVersion, pooler_config, pooler_database, pooler_hba, postgres_version, role_list, + watch_database, PostgresDatabaseNode, Role, Tuning, CREDENTIAL_PATH, CREDENTIAL_STORED_PATH, HEALTH_PATH, LEAST_MEMORY_MIB, }; @@ -26,7 +26,9 @@ pub fn tests() -> Vec { NodeTest::fake("waits_for_postgres_to_answer_for_itself", ready), NodeTest::fake("every_client_comes_in_through_the_pooler", one_door), NodeTest::basic("the_pooler_keeps_sessions_and_its_connections_to_what_postgres_serves", pooler_settings), - NodeTest::basic("only_the_program_s_role_signs_in_through_the_pooler", pooler_sign_ins), + NodeTest::basic("only_the_program_s_role_and_the_login_roles_sign_in_through_the_pooler", pooler_sign_ins), + NodeTest::basic("a_role_name_a_program_cannot_write_or_one_kept_for_postgres_is_refused", role_names), + NodeTest::fake("each_listed_role_is_handed_to_the_credential_server_and_the_pooler", roles), NodeTest::basic("any_database_name_is_kept_as_postgres_has_it", database_names), NodeTest::basic("a_name_the_pooler_cannot_read_gets_a_door_of_its_own", doors), NodeTest::fake("a_database_named_with_a_dash_is_reached_by_that_name", dashed), @@ -502,10 +504,12 @@ fn pooler_settings() -> WeftResult<()> { let config = lines.join("\n"); let line = |key: &str| lines.iter().find(|l| l.starts_with(&format!("{key} = "))).cloned().unwrap_or_else(|| panic!("{key} in {config}")); assert_eq!(line("pool_mode"), "pool_mode = session"); - assert_eq!(line("shop"), format!("shop = host=127.0.0.1 port=5433 dbname='shop' pool_size={}", tuning.query_pool)); + // Every role's pool of a door together is held to the door's share. + let (query, watch) = (tuning.query_pool, tuning.watch_pool); + assert_eq!(line("shop"), format!("shop = host=127.0.0.1 port=5433 dbname='shop' pool_size={query} max_db_connections={query}")); assert_eq!( line(&watch_database("shop")), - format!("shop_watch = host=127.0.0.1 port=5433 dbname='shop' pool_size={}", tuning.watch_pool) + format!("shop_watch = host=127.0.0.1 port=5433 dbname='shop' pool_size={watch} max_db_connections={watch}") ); assert_eq!(line("listen_port"), "listen_port = 5432"); assert_eq!(line("query_wait_timeout"), "query_wait_timeout = 0", "a client waits its turn, never refused for waiting"); @@ -517,23 +521,93 @@ fn pooler_settings() -> WeftResult<()> { assert!(!config.to_lowercase().contains("password"), "{config}"); // Handed over as the container's arguments, a line each, since no // platform carries a line break in an environment variable. - let args = super::pooler_args("shop", &tuning); + let args = super::pooler_args("shop", &tuning, &[]); assert_eq!(args[0], "--config"); assert_eq!(args[1..=lines.len()], lines[..]); assert_eq!(args[lines.len() + 1], "--hba"); - assert_eq!(args[lines.len() + 2..], pooler_hba("shop")[..]); + assert_eq!(args[lines.len() + 2..], pooler_hba("shop", &[])[..]); assert!(args.iter().all(|a| !a.contains('\n'))); Ok(()) } -/// The program's role, by password, to the program's database and its -/// watching door only; the superuser only to the pooler's console, over -/// its unix socket. +/// A role of the `roles` input, as the input writes it. +fn role(name: &str, login: bool, bypass_row_level_security: bool) -> Role { + Role { name: name.into(), login, bypass_row_level_security } +} + +/// The program's role and each login role, each by its own password, to +/// the program's database and its watching door only; a role that does +/// not sign in has no line; the superuser only to the pooler's console, +/// over its unix socket. No password is ever in the rules. fn pooler_sign_ins() -> WeftResult<()> { assert_eq!( - pooler_hba("shop"), + pooler_hba("shop", &[]), ["local pgbouncer weft trust", "host shop,shop_watch app 0.0.0.0/0 scram-sha-256", "host shop,shop_watch app ::/0 scram-sha-256"] ); + let roles = [role("tenant_app", false, false), role("tenant_site", true, false), role("tenant_admin", false, true)]; + assert_eq!( + pooler_hba("shop", &roles), + [ + "local pgbouncer weft trust", + "host shop,shop_watch app 0.0.0.0/0 scram-sha-256", + "host shop,shop_watch app ::/0 scram-sha-256", + "host shop,shop_watch tenant_site 0.0.0.0/0 scram-sha-256", + "host shop,shop_watch tenant_site ::/0 scram-sha-256", + ] + ); + Ok(()) +} + +/// A role name is one a program writes unquoted, so `set role tenant_app` +/// reaches it; the names this node and Postgres keep, and a name listed +/// twice, are refused naming the role. +fn role_names() -> WeftResult<()> { + // Words Postgres reserves no further than as type or column names take + // `set role` as they are. + let listed = vec![ + role("tenant_app", false, false), + role("_site2", true, true), + role(&"a".repeat(63), false, false), + role("admin", false, false), + role("left", false, false), + ]; + assert_eq!(role_list(listed.clone())?, listed); + for refused in [ + "", "Tenant", "2tenant", "tenant-app", "tenant app", "tenant\napp", &"a".repeat(64), "app", "weft", "public", "none", "all", + "current_user", "current_role", "session_user", "pg_monitor", "pg_tenant", "user", "select", "table", "order", "grant", + ] { + match role_list(vec![role(refused, false, false)]) { + Err(weft::WeftError::Input(message)) => assert!(message.contains(&format!("{refused:?}")), "{message}"), + other => panic!("{refused:?} was not refused: {other:?}"), + } + } + let twice = role_list(vec![role("tenant_app", false, false), role("tenant_app", true, false)]); + assert!(matches!(&twice, Err(weft::WeftError::Input(m)) if m.contains("twice")), "{twice:?}"); + Ok(()) +} + +/// The roles the node lists reach the credential server whole, which makes +/// them on every start, and each login role reaches the pooler's rules; a +/// key the input does not have is refused rather than dropped; none listed +/// is an empty list. +async fn roles(rig: FakeRig) -> WeftResult<()> { + let mut settings = config(); + settings["roles"] = json!([{ "name": "tenant_app" }, { "name": "tenant_site", "login": true, "bypassRowLevelSecurity": true }]); + let listed = rig.run_provision_infra(&PostgresDatabaseNode, settings).await.ok()?; + let spec = listed.infra_spec()?; + assert_eq!( + credential_setting(spec, "WEFT_ROLES"), + r#"[{"name":"tenant_app","login":false,"bypassRowLevelSecurity":false},{"name":"tenant_site","login":true,"bypassRowLevelSecurity":true}]"# + ); + let pooler = spec.units[0].containers.iter().find(|c| c.name == "pooler").expect("declared"); + assert!(pooler.args.contains(&"host app,app_watch tenant_site 0.0.0.0/0 scram-sha-256".to_string()), "{:?}", pooler.args); + assert!(!pooler.args.iter().any(|a| a.contains("tenant_app")), "{:?}", pooler.args); + let none = rig.run_provision_infra(&PostgresDatabaseNode, config()).await.ok()?; + assert_eq!(credential_setting(none.infra_spec()?, "WEFT_ROLES"), "[]"); + let mut misspelled = config(); + misspelled["roles"] = json!([{ "name": "tenant_admin", "bypassRls": true }]); + let failure = rig.run_provision_infra(&PostgresDatabaseNode, misspelled).await.failure()?; + assert!(failure.contains("bypassRls"), "{failure}"); Ok(()) } @@ -571,11 +645,12 @@ fn doors() -> WeftResult<()> { let tuning = Tuning::for_machine("1Gi")?; let lines = pooler_config("it's odd", &tuning); let door = pooler_database("it's odd"); + let pool = tuning.query_pool; assert!( - lines.contains(&format!("{door} = host=127.0.0.1 port=5433 dbname='it''s odd' pool_size={}", tuning.query_pool)), + lines.contains(&format!("{door} = host=127.0.0.1 port=5433 dbname='it''s odd' pool_size={pool} max_db_connections={pool}")), "{lines:?}" ); - assert_eq!(pooler_hba("it's odd")[1], format!("host {door},{door}_watch app 0.0.0.0/0 scram-sha-256")); + assert_eq!(pooler_hba("it's odd", &[])[1], format!("host {door},{door}_watch app 0.0.0.0/0 scram-sha-256")); Ok(()) } diff --git a/crates/weft-broker/src/runtime_storage.rs b/crates/weft-broker/src/runtime_storage.rs index ce9d1b9fd..e8129dc65 100644 --- a/crates/weft-broker/src/runtime_storage.rs +++ b/crates/weft-broker/src/runtime_storage.rs @@ -29,6 +29,7 @@ //! POST /v1/storage/admin/link a download link, for a person, weft or the outside //! POST /v1/storage/admin/wipe-prefix weft rm / weft clean //! POST /v1/storage/admin/sweep-run terminate sweep for one run +//! GET /v1/storage/admin/install-cli/{tree} the weft CLI the install stored for a source tree //! //! A page's upload through a slot (bearer = the dispatcher, which passes //! the page's verbs on from its `/files/upload/...` door; the slot in each @@ -154,6 +155,7 @@ pub fn router() -> Router> { .route("/v1/storage/admin/tenant-usage", post(admin_tenant_usage)) .route("/v1/storage/admin/files/{*key}", delete(admin_delete_file)) .route("/v1/storage/admin/meta/{*key}", get(admin_meta)) + .route("/v1/storage/admin/install-cli/{tree}", get(admin_install_cli)) .route("/v1/storage/admin/link", post(admin_link)) .route("/v1/storage/admin/wipe-prefix", post(admin_wipe_prefix)) .route("/v1/storage/admin/sweep-run", post(admin_sweep_run)) @@ -590,9 +592,10 @@ async fn keep_file( } /// `POST /v1/storage/link`: a link to one of the caller's files for node -/// code (`internal`, straight to the bucket at the workers' address) or -/// for the outside (`external`, [`link_route`]), whether the open -/// internet opens it, and the file it opens. An outside link reads the +/// code (`internal`, straight to the bucket at the workers' address), for +/// the project's containers (`container`) or for the outside +/// (`external`), as [`link_route`] routes it, whether the open internet +/// opens it, and the file it opens. An outside link reads the /// project's visitor address in the same statement that reads the file. async fn link( State(state): State>, @@ -603,7 +606,7 @@ async fn link( let parsed = wall(&caller, &req.key)?; let visitors_of = match req.reach { LinkReach::External => Some(caller_project(&caller)?), - LinkReach::Internal => None, + LinkReach::Internal | LinkReach::Container => None, }; let granted = state.runtime_store.grant_link(&parsed, req.ttl_secs, true, visitors_of).await.map_err(map_err)?; let (audience, public) = link_route(&state, req.reach, granted.visitor_address.clone()).await?; @@ -623,10 +626,18 @@ fn project_uuid(project: &str) -> Result { project.parse().map_err(|_| ApiError::from((StatusCode::BAD_REQUEST, format!("'{project}' is not a project id")))) } -/// [`route_link`] with this install's store, own address and internet -/// address. +/// [`route_link`] with this install's store, its containers' address, +/// own address and internet address. async fn link_route(state: &BrokerState, reach: LinkReach, visitor: Option) -> Result<(PresignAudience, bool), ApiError> { - route_link(reach, state.object_store_public_internet, visitor, &state.own_base_url, || state.internet_base()).await + route_link( + reach, + state.object_store_public_internet, + state.container_files_base.as_deref(), + visitor, + &state.own_base_url, + || state.internet_base(), + ) + .await } /// The audience of a page's upload parts through a slot of `project`: @@ -654,8 +665,12 @@ async fn visitor_address(state: &BrokerState, project: uuid::Uuid) -> Result Result( reach: LinkReach, store_public: bool, + containers: Option<&str>, visitor: Option, own: &str, internet: F, @@ -673,8 +689,10 @@ where F: FnOnce() -> Fut, Fut: std::future::Future>>, { - if reach == LinkReach::Internal { - return Ok((PresignAudience::Install, store_public)); + match (reach, containers) { + (LinkReach::Container, Some(base)) => return Ok((PresignAudience::Forwarded { base: base.to_string() }, false)), + (LinkReach::Internal | LinkReach::Container, _) => return Ok((PresignAudience::Install, store_public)), + (LinkReach::External, _) => {} } if store_public { return Ok((PresignAudience::External, true)); @@ -1278,6 +1296,42 @@ async fn admin_delete_file( /// install as a project's callers do. An outside link's file is checked /// to be one that project's callers reach ([`outside_reader`]), so a /// dispatcher bug can still never hand out another tenant's file. +/// How long a link to the install's stored CLI works: long enough for a +/// deploy to download it on a slow runner. +const INSTALL_CLI_LINK_SECS: u64 = 3600; + +/// `GET /v1/storage/admin/install-cli/{tree}`: a link to the weft CLI the +/// install workflow stored for the source tree `tree`, with its sha256; +/// 404 when it stored none. The install's own file, beside every tenant's +/// and no tenant's, which the workflow writes and replaces, so nothing of +/// it is recorded here. +async fn admin_install_cli( + State(state): State>, + headers: HeaderMap, + Path(tree): Path, +) -> Result, ApiError> { + control_plane(&state, &headers).await?; + if !weft_core::install::is_tree_id(&tree) { + return Err(ApiError::from((StatusCode::BAD_REQUEST, format!("'{tree}' is not a git tree id (40 or 64 lowercase hex digits)")))); + } + // A read and a signature: nothing is written, so a failure is one + // asking again may clear. + let failed = |e: anyhow::Error| map_err(RuntimeStoreError::BeforeWrites(e.context("read the stored weft CLI"))); + let (cli, sha256) = weft_core::install::install_cli_keys(&tree); + let Some(sha256) = state.object_store.get(&sha256).await.map_err(failed)? else { + return Err(ApiError::from((StatusCode::NOT_FOUND, format!("no weft CLI is stored for the source tree {tree}")))); + }; + let link = weft_platform_traits::GetLink { + audience: PresignAudience::External, + signed_at: std::time::SystemTime::now(), + ttl_secs: INSTALL_CLI_LINK_SECS, + content_type: Some("application/octet-stream".into()), + disposition: None, + }; + let url = state.object_store.presign_get(&cli, &link).await.map_err(failed)?; + Ok(Json(weft_core::install::InstallCli { url, sha256: String::from_utf8_lossy(&sha256).trim().to_string() })) +} + async fn admin_link( State(state): State>, headers: HeaderMap, @@ -1360,6 +1414,7 @@ async fn admin_sweep_run( #[cfg(test)] mod link_route_tests { use super::{map_err, route_link, LinkReach, PresignAudience, RuntimeStoreError, StatusCode}; + use weft_platform_traits::object_store::{fake::FakeObjectStore, GetLink, ObjectStore}; use weft_task_store::db; /// What a statement that got no connection fails with. @@ -1442,18 +1497,55 @@ mod link_route_tests { async fn an_internal_link_is_public_only_on_a_public_store() { for store_public in [false, true] { let (read, ask) = internet(tunnel()); - let route = route_link(LinkReach::Internal, store_public, Some("https://shop.example".into()), OWN, ask).await.unwrap(); + let route = route_link(LinkReach::Internal, store_public, None, Some("https://shop.example".into()), OWN, ask).await.unwrap(); assert_eq!(route, (PresignAudience::Install, store_public)); assert!(!read.load(std::sync::atomic::Ordering::SeqCst)); } } + /// A local install's internal port as its containers reach it. + const CONTAINERS: &str = "http://host.docker.internal:14113"; + + /// A link for the project's containers on a local install goes + /// through the files door on the internal port as a container reaches + /// it, signed for that address (a container's `127.0.0.1` is itself), + /// and the internet does not open it. On an install whose containers + /// reach the store where the workers do (a cloud one), it is the + /// internal link, whatever the store. Neither asks the tunnel. + #[tokio::test] + async fn a_container_link_goes_through_the_containers_address_else_it_is_the_internal_one() { + let (read, ask) = internet(tunnel()); + let local = route_link(LinkReach::Container, false, Some(CONTAINERS), None, OWN, ask).await.unwrap(); + assert_eq!(local, (forwarded(CONTAINERS), false)); + let signed = FakeObjectStore::new() + .presign_get( + "runtime/t/k", + &GetLink { + audience: local.0, + signed_at: std::time::UNIX_EPOCH, + ttl_secs: 900, + content_type: None, + disposition: None, + }, + ) + .await + .unwrap(); + assert!(signed.starts_with("http://host.docker.internal:14113/files/"), "{signed}"); + for store_public in [false, true] { + let (_, ask) = internet(tunnel()); + let internal = route_link(LinkReach::Internal, store_public, None, None, OWN, ask).await.unwrap(); + let (_, ask) = internet(tunnel()); + assert_eq!(route_link(LinkReach::Container, store_public, None, None, OWN, ask).await.unwrap(), internal); + } + assert!(!read.load(std::sync::atomic::Ordering::SeqCst)); + } + /// A store the internet reaches signs its own link, whatever else the /// install has. #[tokio::test] async fn an_outside_link_on_a_public_store_is_the_stores() { let (read, ask) = internet(tunnel()); - let route = route_link(LinkReach::External, true, Some("https://shop.example".into()), OWN, ask).await.unwrap(); + let route = route_link(LinkReach::External, true, None, Some("https://shop.example".into()), OWN, ask).await.unwrap(); assert_eq!(route, (PresignAudience::External, true)); assert!(!read.load(std::sync::atomic::Ordering::SeqCst)); } @@ -1462,7 +1554,7 @@ mod link_route_tests { #[tokio::test] async fn an_outside_link_goes_through_the_visitor_address_without_asking_the_tunnel() { let (read, ask) = internet(Err(anyhow::anyhow!("never asked"))); - let route = route_link(LinkReach::External, false, Some("https://shop.example/weft".into()), OWN, ask).await.unwrap(); + let route = route_link(LinkReach::External, false, None, Some("https://shop.example/weft".into()), OWN, ask).await.unwrap(); assert_eq!(route, (forwarded("https://shop.example/weft"), true)); assert!(!read.load(std::sync::atomic::Ordering::SeqCst)); } @@ -1472,10 +1564,10 @@ mod link_route_tests { #[tokio::test] async fn an_outside_link_without_a_site_goes_through_the_internet_else_the_own_address() { let (read, ask) = internet(tunnel()); - assert_eq!(route_link(LinkReach::External, false, None, OWN, ask).await.unwrap(), (forwarded("https://t.example"), true)); + assert_eq!(route_link(LinkReach::External, false, None, None, OWN, ask).await.unwrap(), (forwarded("https://t.example"), true)); assert!(read.load(std::sync::atomic::Ordering::SeqCst)); let (_, ask) = internet(Ok(None)); - assert_eq!(route_link(LinkReach::External, false, None, OWN, ask).await.unwrap(), (forwarded(OWN), false)); + assert_eq!(route_link(LinkReach::External, false, None, None, OWN, ask).await.unwrap(), (forwarded(OWN), false)); } /// A tunnel that cannot say its address refuses the link: one through @@ -1483,7 +1575,7 @@ mod link_route_tests { #[tokio::test] async fn an_unreadable_tunnel_refuses_the_outside_link() { let (_, ask) = internet(Err(anyhow::anyhow!("cloudflared is down"))); - let refused = route_link(LinkReach::External, false, None, OWN, ask).await.unwrap_err(); + let refused = route_link(LinkReach::External, false, None, None, OWN, ask).await.unwrap_err(); assert_eq!(refused.status, StatusCode::SERVICE_UNAVAILABLE); } diff --git a/crates/weft-broker/src/state.rs b/crates/weft-broker/src/state.rs index 45e2fd978..e132b34d0 100644 --- a/crates/weft-broker/src/state.rs +++ b/crates/weft-broker/src/state.rs @@ -102,6 +102,8 @@ pub struct BrokerState { /// neither a site nor the internet carries it (the network address /// when one is set): such a link says `public: false`. pub own_base_url: String, + /// See [`BrokerSettings::container_files_base`]. + pub container_files_base: Option, /// True iff the object store's own presigned links are reachable from /// the open internet (Cloud Storage): a link to a stored file for the /// outside is then the store's, straight, and never goes through the @@ -135,6 +137,10 @@ pub struct BrokerSettings { /// is the address, read live. pub internet: weft_platform_traits::config::InternetAddress, pub own_base_url: String, + /// Where the project's own containers reach the install's files door + /// (`InstallConfig::container_files_base`), when they do not reach the + /// store at the workers' address: a link for them goes through it. + pub container_files_base: Option, pub object_store_public_internet: bool, pub slot_key: weft_core::storage::slot::SlotKey, /// How long this process's platform lets one line live before cutting @@ -222,6 +228,7 @@ impl BrokerState { app_provider: settings.app_provider, address, own_base_url: settings.own_base_url.trim_end_matches('/').to_string(), + container_files_base: settings.container_files_base.map(|base| base.trim_end_matches('/').to_string()), object_store_public_internet: settings.object_store_public_internet, slot_key: settings.slot_key, supervisor_booking: settings.supervisor_booking, diff --git a/crates/weft-broker/tests/db/install_cli.rs b/crates/weft-broker/tests/db/install_cli.rs new file mode 100644 index 000000000..17bb24616 --- /dev/null +++ b/crates/weft-broker/tests/db/install_cli.rs @@ -0,0 +1,45 @@ +//! The weft CLI the install workflow stored, through the broker's real +//! router: weft's own account gets a link to it with its sha256, and a +//! tree with none stored, or a name that is no tree, is refused plainly. + +use std::sync::Arc; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use weft_platform_traits::object_store::fake::FakeObjectStore; +use weft_platform_traits::ObjectStore as _; +use weft_task_store::db::Pool; + +use crate::old_shapes::{answer, broker_state_on, CORE_TOKEN}; + +fn asked(tree: &str) -> Request { + Request::get(format!("/v1/storage/admin/install-cli/{tree}")) + .header("authorization", format!("Bearer {CORE_TOKEN}")) + .header(weft_platform_traits::identity::REPLICA_HEADER, "dispatcher-1") + .header(weft_platform_traits::identity::ROLE_HEADER, "dispatcher") + .body(Body::empty()) + .unwrap() +} + +#[weft_db_macros::test] +async fn the_stored_cli_is_linked_with_its_sha256_and_a_missing_one_is_a_404(pool: Pool) { + let bucket = Arc::new(FakeObjectStore::new()); + let tree = "ab".repeat(20); + let (cli, sha256) = weft_core::install::install_cli_keys(&tree); + bucket.put(&cli, bytes::Bytes::from_static(b"the cli")).await.unwrap(); + bucket.put(&sha256, bytes::Bytes::from_static(b"5eed\n")).await.unwrap(); + let router = weft_broker::router(broker_state_on(pool, bucket).await); + + let (status, body) = answer(&router, asked(&tree)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + let stored: weft_core::install::InstallCli = serde_json::from_str(&body).unwrap(); + assert_eq!(stored.sha256, "5eed", "read as written, without its line break"); + assert!(stored.url.contains(&cli), "{}", stored.url); + + let (status, body) = answer(&router, asked(&"cd".repeat(20))).await; + assert_eq!(status, StatusCode::NOT_FOUND, "{body}"); + assert!(body.contains("no weft CLI is stored"), "{body}"); + + let (status, body) = answer(&router, asked("..%2Fsecret")).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); +} diff --git a/crates/weft-broker/tests/db/main.rs b/crates/weft-broker/tests/db/main.rs index 999577b4b..a2d1c9f1e 100644 --- a/crates/weft-broker/tests/db/main.rs +++ b/crates/weft-broker/tests/db/main.rs @@ -8,6 +8,7 @@ mod caller_auth; mod door_let_go; mod held_signals; +mod install_cli; mod lifecycle_writes; mod old_shapes; mod program_tokens; diff --git a/crates/weft-broker/tests/db/old_shapes.rs b/crates/weft-broker/tests/db/old_shapes.rs index 7be3faccb..9ef303b7b 100644 --- a/crates/weft-broker/tests/db/old_shapes.rs +++ b/crates/weft-broker/tests/db/old_shapes.rs @@ -13,7 +13,7 @@ use weft_platform_traits::identity::{IdentityRefused, Principal}; use weft_task_store::db::Pool; const WORKER_TOKEN: &str = "worker-token"; -const CORE_TOKEN: &str = "core-token"; +pub(crate) const CORE_TOKEN: &str = "core-token"; /// One project's worker, for its one token, and weft's own account. struct OneWorker; @@ -34,6 +34,12 @@ async fn broker(pool: Pool) -> axum::Router { } async fn broker_state(pool: Pool) -> Arc { + broker_state_on(pool, Arc::new(weft_platform_traits::object_store::fake::FakeObjectStore::new())).await +} + +/// The broker's state on `object_store`, for a test that puts objects in +/// the bucket first. +pub(crate) async fn broker_state_on(pool: Pool, object_store: Arc) -> Arc { let signals = weft_task_store::pg_signal::PgSignalWatch::start( pool.config(), weft_broker::state::BROKER_CHANNELS, @@ -49,13 +55,14 @@ async fn broker_state(pool: Pool) -> Arc { weft_broker::state::BrokerSettings { auth: weft_broker::AuthConfig { audiences: vec!["http://broker".into()] }, identity: Arc::new(OneWorker), - object_store: Arc::new(weft_platform_traits::object_store::fake::FakeObjectStore::new()), + object_store, entitlements: Arc::new(weft_broker::entitlement::LocalEntitlementSource::from_env()), credentials: Arc::new(weft_broker::credential::FileCredentialSource::from_env()), app_provider: Arc::new(weft_broker::app_provider::FileAppProvider::from_env()), service_url: "http://broker".into(), internet: weft_platform_traits::config::InternetAddress::None, own_base_url: "http://broker".into(), + container_files_base: None, object_store_public_internet: false, slot_key: weft_core::storage::slot::SlotKey::of(b"a test secret"), line_max_age: None, @@ -87,7 +94,7 @@ fn old_caller_verify(bearer: &str) -> Request { .unwrap() } -async fn answer(router: &axum::Router, request: Request) -> (StatusCode, String) { +pub(crate) async fn answer(router: &axum::Router, request: Request) -> (StatusCode, String) { let response = router.clone().oneshot(request).await.unwrap(); let status = response.status(); let body = axum::body::to_bytes(response.into_body(), usize::MAX).await.unwrap(); diff --git a/crates/weft-cli/src/commands/daemon.rs b/crates/weft-cli/src/commands/daemon.rs index f93b94991..20dc82f65 100644 --- a/crates/weft-cli/src/commands/daemon.rs +++ b/crates/weft-cli/src/commands/daemon.rs @@ -1040,16 +1040,6 @@ fn container_internal_url(ports: Ports) -> String { format!("http://host.docker.internal:{}", ports.internal) } -/// Where the internal port listens. An infra unit's agent reaches it from -/// its container through the Docker bridge, which on Linux is not loopback, -/// so it listens on every interface there (every internal route checks -/// its caller's identity). Docker Desktop brings a container's calls to -/// the machine's loopback, so it stays on loopback there. -fn internal_listen(ports: Ports) -> std::net::SocketAddr { - let host = if cfg!(target_os = "linux") { [0, 0, 0, 0] } else { [127, 0, 0, 1] }; - (host, ports.internal).into() -} - /// The edge settings: an environment variable when one is set, what the /// install already had otherwise, the defaults on a new install. fn edge_config(kept: Option) -> Result { @@ -1253,7 +1243,10 @@ fn install_config( worker_idle_stop_seconds: idle_stop, listen: Listen { public: ([127, 0, 0, 1], ports.public).into(), - internal: internal_listen(ports), + // Also where the project's containers reach this machine, + // which the runtime asks Docker on every start + // (`weft_platform_local::docker::containers_call_in_at`). + internal: ([127, 0, 0, 1], ports.internal).into(), // Served whether or not a tunnel carries it: it answers a // subset of what the public port does, and a tunnel opened // later finds it already there. @@ -1567,6 +1560,23 @@ fn launchd_plist_path(install: &Install) -> PathBuf { home.join("Library/LaunchAgents").join(format!("{}.plist", launchd_label(install))) } +/// The launchd job that runs `line` under `label`. What the shell prints +/// before the runtime's own redirect takes over (a secrets file it cannot +/// read, a binary gone) goes to `log` through `StandardErrorPath`. +fn launchd_plist(label: &str, line: &str, log: &str) -> String { + format!( + "\n\n\nLabel{label}\nProgramArguments/bin/sh-c{line}\nStandardErrorPath{log}\nRunAtLoad\nKeepAlive\n\n", + label = xml_text(label), + line = xml_text(line), + log = xml_text(log), + ) +} + +/// `text` as a plist's `` holds it. +fn xml_text(text: &str) -> String { + text.replace('&', "&").replace('<', "<").replace('>', ">") +} + fn pid_path(install: &Install) -> PathBuf { install.dir.join("runtime.pid") } @@ -1576,7 +1586,8 @@ fn pid_path(install: &Install) -> PathBuf { /// runtime's builds, needs checks and workers use the person's own tools: /// cargo, python3, Homebrew's), the secrets into the environment, then the /// runtime, which keeps its log bounded itself (`--log`); what escapes it -/// (a panic's last words) is appended to the same file. +/// (a panic's last words) is appended to the same file, which the runtime +/// keeps its stderr on across each roll (`weft_runtime::log_file`). fn runtime_command(install: &Install, bin: &Path) -> String { let path = std::env::var("PATH").unwrap_or_default().replace('\'', "'\\''"); format!( @@ -1591,10 +1602,26 @@ fn runtime_command(install: &Install, bin: &Path) -> String { async fn run_checked(cmd: &mut Command, what: &str) -> Result<()> { let out = cmd.output().await.with_context(|| format!("run {what}"))?; - anyhow::ensure!(out.status.success(), "{what} failed: {}", String::from_utf8_lossy(&out.stderr).trim()); + anyhow::ensure!(out.status.success(), "{}", failed(what, &out)); Ok(()) } +/// A service manager's failure, whole: its exit status and everything it +/// printed. `launchctl` says why on stdout as often as on stderr, and an +/// error that kept only stderr once read `launchctl bootstrap failed: ` +/// and nothing more. +fn failed(what: &str, out: &std::process::Output) -> String { + let said: Vec = [&out.stdout, &out.stderr] + .into_iter() + .map(|bytes| String::from_utf8_lossy(bytes).trim().to_string()) + .filter(|text| !text.is_empty()) + .collect(); + match said.is_empty() { + true => format!("{what} failed ({}) and printed nothing", out.status), + false => format!("{what} failed ({}): {}", out.status, said.join("\n")), + } +} + async fn start_runtime(install: &Install) -> Result<()> { let bin = runtime_binary()?; let line = runtime_command(install, &bin); @@ -1623,15 +1650,10 @@ async fn start_runtime(install: &Install) -> Result<()> { let plist = launchd_plist_path(install); std::fs::create_dir_all(plist.parent().expect("a plist path has a directory"))?; let label = launchd_label(install); - let escaped = line.replace('&', "&").replace('<', "<").replace('>', ">"); - std::fs::write( - &plist, - format!( - "\n\n\nLabel{label}\nProgramArguments/bin/sh-c{escaped}\nRunAtLoad\nKeepAlive\n\n" - ), - )?; + std::fs::write(&plist, launchd_plist(&label, &line, &install.log_path().display().to_string()))?; + // Booted out already: the start stops the runtime before it + // gets here (`stop_runtime`), and says so when it could not. let domain = format!("gui/{}", current_uid().await?); - let _ = Command::new("launchctl").args(["bootout", &format!("{domain}/{label}")]).output().await; run_checked(Command::new("launchctl").args(["bootstrap", &domain, &plist.display().to_string()]), "launchctl bootstrap").await } Manager::Detached => { @@ -1708,15 +1730,17 @@ async fn stop_runtime(install: &Install) -> Result<()> { // still active after a failed stop is not. if !out.status.success() { let active = Command::new("systemctl").args(["--user", "is-active", "--quiet", &name]).status().await?.success(); - anyhow::ensure!(!active, "systemctl --user stop {name} failed: {}", String::from_utf8_lossy(&out.stderr).trim()); + anyhow::ensure!(!active, "{}", failed(&format!("systemctl --user stop {name}"), &out)); } } Manager::Launchd => { let target = format!("gui/{}/{}", current_uid().await?, launchd_label(install)); let out = Command::new("launchctl").args(["bootout", &target]).output().await.context("run launchctl")?; + // A service that was not loaded is already stopped; one still + // loaded after a failed bootout is not. if !out.status.success() { let loaded = Command::new("launchctl").args(["print", &target]).output().await?.status.success(); - anyhow::ensure!(!loaded, "launchctl bootout {target} failed: {}", String::from_utf8_lossy(&out.stderr).trim()); + anyhow::ensure!(!loaded, "{}", failed(&format!("launchctl bootout {target}"), &out)); } } Manager::Detached => stop_detached(install).await?, @@ -2070,6 +2094,32 @@ fn which(ctx: &Ctx, wanted: &Wanted) -> Result<()> { mod tests { use super::*; + /// The job's command line and log path reach launchd as written, with + /// every character XML gives a meaning escaped. + #[test] + fn the_launchd_job_carries_its_command_and_log_escaped() { + let plist = launchd_plist("ai.weavemind.weft", "a && b < c > d", "/Users/x/R&D <1>/runtime.log"); + assert!(plist.contains("a && b < c > d"), "{plist}"); + assert!(plist.contains("StandardErrorPath/Users/x/R&D <1>/runtime.log"), "{plist}"); + assert!(!plist.contains("R&D"), "{plist}"); + } + + /// A service manager's failure keeps its status and what it printed on + /// either stream, and says so when it printed nothing. + #[test] + fn a_failed_service_manager_is_quoted_whole() { + use std::os::unix::process::ExitStatusExt; + let out = |stdout: &str, stderr: &str| std::process::Output { + status: std::process::ExitStatus::from_raw(5 << 8), + stdout: stdout.as_bytes().to_vec(), + stderr: stderr.as_bytes().to_vec(), + }; + let both = failed("launchctl bootstrap", &out("Bootstrap failed: 5: Input/output error\n", " try again\n")); + assert!(both.contains("exit status: 5"), "{both}"); + assert!(both.ends_with("Bootstrap failed: 5: Input/output error\ntry again"), "{both}"); + assert!(failed("launchctl bootstrap", &out("", "")).ends_with("and printed nothing")); + } + /// Docker's refusal for a container whose network was made again is /// told apart from every other start failure. #[test] diff --git a/crates/weft-cli/src/commands/infra_card.rs b/crates/weft-cli/src/commands/infra_card.rs index e292e816a..6f6af3de1 100644 --- a/crates/weft-cli/src/commands/infra_card.rs +++ b/crates/weft-cli/src/commands/infra_card.rs @@ -39,9 +39,10 @@ impl Card { serde_json::from_value::(answer).context("read the node's display") } - /// Press `action` and hand back what the node answered. - pub async fn press(&self, action: &LiveAction) -> Result { - self.client.post_json(&format!("{}/action{}", self.base, self.query), &serde_json::to_value(action.press())?).await + /// Send `press` (a button's, or a reader's `taken`) and hand back what + /// the node answered. + pub async fn press(&self, press: &weft_core::live::LivePress) -> Result { + self.client.post_json(&format!("{}/action{}", self.base, self.query), &serde_json::to_value(press)?).await } } @@ -172,7 +173,7 @@ pub async fn run_press(ctx: Ctx, args: PressArgs) -> Result<()> { // Naming the button's action on the command line is the choice the // editor's confirm box asks for, so the press never asks again; the // button's warning is printed with the result instead. - let answer = card.press(action).await?; + let answer = card.press(&action.press()).await?; // A node that answers a press with the value its card shows as a // secret does not get it printed. The mask comes from the card as it // was read before the press: once the press is done, nothing here may diff --git a/crates/weft-cli/src/commands/infra_env.rs b/crates/weft-cli/src/commands/infra_env.rs index d14db9b16..c991fda9e 100644 --- a/crates/weft-cli/src/commands/infra_env.rs +++ b/crates/weft-cli/src/commands/infra_env.rs @@ -8,9 +8,11 @@ //! Generic over nodes: the card is whatever the node's display (`/live`) //! serves. `--set NAME=Label` takes the item under that label, whatever //! its kind; `--as NAME` takes the card's only `secret` item. A node that -//! hands its secret out once replaces that item with a line carrying a -//! button (the Postgres database's "Reset password"); this refuses then -//! and points at `weft infra press`, which presses it. +//! hands its secret out once names what its reader presses once it holds +//! it (`LiveItem::taken`), which this presses as soon as the file is +//! written; from then on the node shows a line it marks handed over +//! (`LiveItem::handed_over`), carrying a button, in its place (the Postgres database's "Reset password"), and this refuses and +//! points at `weft infra press`, which presses it. use std::io::Write; use std::path::{Path, PathBuf}; @@ -92,11 +94,10 @@ enum Pick<'a> { /// Find an item in a card. /// -/// A secret handed over already shows as another kind of item under the -/// same label, carrying the button that makes it readable again, so a -/// non-secret line with a button is that state, never a value to write. -/// With no label, that item is recognized as the only one with a button, -/// so a node with one secret works with `--as` whichever state it is in. +/// A secret the node handed over already shows as a line the node marks +/// so (`LiveItem::handed_over`), under the same label, carrying the button +/// that makes a new one: that state is never a value to write. Any other +/// line is its value, a plain line with a button included. fn pick<'a>(feed: &'a LiveFeed, want: &Want) -> Result> { let labels = |items: &[&LiveItem]| { items.iter().map(|i| format!("'{}'", i.label)).collect::>().join(", ") @@ -104,29 +105,21 @@ fn pick<'a>(feed: &'a LiveFeed, want: &Want) -> Result> { let all: Vec<&LiveItem> = feed.items.iter().collect(); match want { Want::Label(label) => match feed.items.iter().find(|i| &i.label == label) { - Some(item) => match (&item.kind, &item.action) { - (LiveItemKind::Secret, _) | (_, None) => Ok(Pick::Value(item)), - (_, Some(action)) => Ok(Pick::HandedOver(item, action)), - }, + Some(item) => as_picked(item), None if all.is_empty() => bail!("the node shows nothing, so there is no '{label}' to write"), None => bail!("the node shows no '{label}'; it shows {}", labels(&all)), }, + // Every secret the card holds counts, whether it shows its value + // or has been handed over: with two, the one showing now is not + // necessarily the one asked for, so `--as` names neither and the + // line is named instead. Want::OnlySecret => { - let secrets: Vec<&LiveItem> = feed.items.iter().filter(|i| i.kind == LiveItemKind::Secret).collect(); + let secrets: Vec<&LiveItem> = + feed.items.iter().filter(|i| i.kind == LiveItemKind::Secret || i.handed_over).collect(); match secrets.as_slice() { - [only] => Ok(Pick::Value(only)), - [] => { - let buttons: Vec<(&LiveItem, &LiveAction)> = - feed.items.iter().filter_map(|i| Some((i, i.action.as_ref()?))).collect(); - match buttons.as_slice() { - [(item, action)] => Ok(Pick::HandedOver(item, action)), - _ if all.is_empty() => bail!("the node shows nothing, so there is no secret to write"), - _ => bail!( - "the node shows no secret right now; name the line with --set NAME=Label (it shows {})", - labels(&all) - ), - } - } + [only] => as_picked(only), + [] if all.is_empty() => bail!("the node shows nothing, so there is no secret to write"), + [] => bail!("the node shows no secret right now; name the line with --set NAME=Label (it shows {})", labels(&all)), many => bail!("the node shows several secrets; name each with --set NAME=Label ({})", labels(many)), } } @@ -134,6 +127,21 @@ fn pick<'a>(feed: &'a LiveFeed, want: &Want) -> Result> { } } +/// `item` as what it gives: its value, or, for a secret the node handed +/// over, the button that makes a new one. One marked handed over with no +/// button leaves nothing to press, which is the node's mistake to name. +fn as_picked(item: &LiveItem) -> Result> { + match (item.handed_over, &item.action) { + (false, _) => Ok(Pick::Value(item)), + (true, Some(action)) => Ok(Pick::HandedOver(item, action)), + (true, None) => bail!( + "the node shows '{}' as a secret it handed over, with no button to get a new one; its card says \"{}\"", + item.label, + item_text(item) + ), + } +} + /// The address the door for `endpoint` of the copy at `place` (`instance` /// `None`: the shared one) answers at, read off the project's doors. fn door_address(doors: &weft_core::infra::wire::DoorsResponse, place: &str, instance: Option<&weft_core::instance::InstanceId>, endpoint: &str) -> Result { @@ -292,6 +300,9 @@ pub async fn run(ctx: Ctx, args: EnvArgs) -> Result<()> { }; let mut pairs = Vec::new(); let mut written = Vec::new(); + // The secrets their node hands out once, retired once the file holds + // them. + let mut taken = Vec::new(); for (name, want) in &wanted { if let (Want::Door(endpoint), Some(doors)) = (want, &doors) { let address = door_address(doors, &card.place, args.instance.as_ref(), endpoint)?; @@ -304,6 +315,7 @@ pub async fn run(ctx: Ctx, args: EnvArgs) -> Result<()> { Pick::HandedOver(item, action) => bail!("{}", handed_over(&card.place, item, action, ctx.on())), }; let value = item_text(item); + taken.extend(item.taken_press()); written.push(match item.kind { LiveItemKind::Secret => (name.clone(), None), _ => (name.clone(), Some(value.clone())), @@ -311,6 +323,17 @@ pub async fn run(ctx: Ctx, args: EnvArgs) -> Result<()> { pairs.push((name.clone(), value)); } write_env(&path, &pairs)?; + for press in &taken { + card.press(press).await.with_context(|| { + format!( + "{} is written, but {} refused to mark the secret it holds as taken (the cause is below: when the node \ + made a new one meanwhile, the file holds the old one); run this again, which writes what the card \ + shows now and retires it", + args.into.display(), + card.place + ) + })?; + } let shown = args.into.display().to_string(); let report = serde_json::json!({ "file": shown, @@ -369,7 +392,7 @@ mod tests { fn handed_over() -> LiveFeed { LiveFeed::new(vec![ LiveItem::text("User", "weft"), - LiveItem::text("Password", "handed over").with_action(reset_button()), + LiveItem::text("Password", "handed over").as_handed_over().with_action(reset_button()), ]) } @@ -460,6 +483,34 @@ mod tests { assert_eq!(pick(&feed, &label("B")).unwrap(), Pick::Value(&feed.items[1])); } + /// A plain line with a button is its value, never a secret handed + /// over (a node's restart button on its status line, say), and it is + /// no secret for `--as` either. + #[test] + fn a_plain_line_with_a_button_is_its_value() { + let feed = LiveFeed::new(vec![ + LiveItem::text("Shell", "running").with_action(LiveAction::new("Restart the shell", "restart_shell")), + LiveItem::secret("Key", "k"), + ]); + assert_eq!(pick(&feed, &label("Shell")).unwrap(), Pick::Value(&feed.items[0])); + assert_eq!(pick(&feed, &Want::OnlySecret).unwrap(), Pick::Value(&feed.items[1])); + let lost = LiveFeed::new(vec![LiveItem::text("Password", "gone").as_handed_over()]); + assert!(pick(&lost, &label("Password")).unwrap_err().to_string().contains("no button")); + } + + /// A secret handed over still counts: `--as` never writes the other + /// one just because it is the only one showing its value. + #[test] + fn a_handed_over_secret_beside_another_one_needs_a_label() { + let feed = LiveFeed::new(vec![ + LiveItem::text("Password", "handed over").as_handed_over().with_action(reset_button()), + LiveItem::secret("site password", "2").with_action(LiveAction::new("Reset password", "reset_password:site")), + ]); + let err = pick(&feed, &Want::OnlySecret).unwrap_err().to_string(); + assert!(err.contains("'Password', 'site password'"), "{err}"); + assert_eq!(pick(&feed, &label("site password")).unwrap(), Pick::Value(&feed.items[1])); + } + #[test] fn an_unknown_label_lists_the_ones_there_are() { let feed = fresh(); diff --git a/crates/weft-cli/templates/ci/gcp.yml b/crates/weft-cli/templates/ci/gcp.yml index e6cdb6ddf..4d728affe 100644 --- a/crates/weft-cli/templates/ci/gcp.yml +++ b/crates/weft-cli/templates/ci/gcp.yml @@ -33,7 +33,9 @@ concurrency: jobs: deploy: # A named release: `ubuntu-latest` moves to a new one on GitHub's - # schedule, under a workflow that worked the day before. + # schedule, under a workflow that worked the day before. The same as + # the install workflow's, whose CLI this job downloads and runs. + # SYNC: runs-on <-> .github/workflows/install-gcp.yml (runs-on, store the weft CLI in the install) runs-on: ubuntu-24.04 env: # What happens to the runs going when the deploy replaces what they @@ -54,10 +56,11 @@ jobs: # so an install upgraded since `weft target export` is followed here # with no setting to change. The CLI below is that weft's own (it # reads the project's nodes from this source), so what it sends is - # what the install expects. weft's release of that CLI is downloaded - # (waited for while it is still being built), otherwise it is built - # here from a cache that keeps the dependencies between runs, so a - # newer weft compiles only what changed in it. + # what the install expects. It is downloaded from the install itself, + # which stores the CLI its install workflow used, else from weft's + # release of that source (waited for while it is still being built), + # otherwise it is built here from a cache that keeps the dependencies + # between runs, so a newer weft compiles only what changed in it. - name: ask the install which weft it runs id: source run: | @@ -82,7 +85,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - if .weft/scripts/release-cli.sh .weft "$RUNNER_TEMP/weft"; then + if .weft/scripts/release-cli.sh .weft "$RUNNER_TEMP/weft" "${{ vars.WEFT_PUBLIC_URL }}"; then echo "$RUNNER_TEMP/weft" >> "$GITHUB_PATH" echo "found=yes" >> "$GITHUB_OUTPUT" else diff --git a/crates/weft-core/src/content_cache.rs b/crates/weft-core/src/content_cache.rs index 551d85c0a..9bbcfc7f2 100644 --- a/crates/weft-core/src/content_cache.rs +++ b/crates/weft-core/src/content_cache.rs @@ -8,6 +8,11 @@ //! still in use names it. A copy of rows that do change //! (`weft_task_store::held_copy`) keeps them here too, and drops an entry //! (`forget`) the moment its rows change. +//! +//! A cache is bounded by how many entries it holds ([`ContentCache::new`]) +//! when its values are about the same size, and by how many bytes they +//! weigh ([`ContentCache::weighed`]) when they are not: a program is a few +//! kilobytes or tens of megabytes, so a count of them bounds nothing. use std::hash::Hash; use std::sync::{Arc, Mutex}; @@ -16,35 +21,86 @@ use std::sync::{Arc, Mutex}; /// project and definition hash, what a definition declares by its digest, /// a held copy's rows by whose they are. pub struct ContentCache { - entries: Mutex>>, + entries: Mutex>, + /// What one value weighs, in bytes, and how many bytes all of them may + /// weigh together; `None` for a cache bounded by its entries alone. + weight: Option>, +} + +struct Entries { + held: lru::LruCache>, + /// What every held value weighs together. + bytes: usize, +} + +struct Held { + value: Arc, + bytes: usize, +} + +struct Weight { + of: fn(&V) -> usize, + limit: usize, } impl ContentCache { /// A cache holding at most `capacity` entries. pub fn new(capacity: usize) -> Self { let capacity = std::num::NonZeroUsize::new(capacity).expect("a content cache holds at least one entry"); - Self { entries: Mutex::new(lru::LruCache::new(capacity)) } + Self { entries: Mutex::new(Entries { held: lru::LruCache::new(capacity), bytes: 0 }), weight: None } + } + + /// A cache whose values weigh at most `limit` bytes together, each + /// weighed once by `weigh` as it is put. A value heavier than `limit` + /// on its own is never kept. + pub fn weighed(limit: usize, weigh: fn(&V) -> usize) -> Self { + Self { entries: Mutex::new(Entries { held: lru::LruCache::unbounded(), bytes: 0 }), weight: Some(Weight { of: weigh, limit }) } } /// The value held under `key`, if this process has it. pub fn get(&self, key: &K) -> Option> { - self.entries.lock().expect("content cache").get(key).cloned() + self.entries.lock().expect("content cache").held.get(key).map(|held| held.value.clone()) } - /// Keep `value` as what is held under `key`. + /// Keep `value` as what is held under `key`, dropping the least + /// recently used until the cache is within its bound again. pub fn put(&self, key: K, value: Arc) { - self.entries.lock().expect("content cache").put(key, value); + let bytes = self.weight.as_ref().map_or(0, |weight| (weight.of)(&value)); + let mut entries = self.entries.lock().expect("content cache"); + let Some(weight) = &self.weight else { + entries.held.put(key, Held { value, bytes }); + return; + }; + if bytes > weight.limit { + if let Some(gone) = entries.held.pop(&key) { + entries.bytes -= gone.bytes; + } + return; + } + if let Some(replaced) = entries.held.put(key, Held { value, bytes }) { + entries.bytes -= replaced.bytes; + } + entries.bytes += bytes; + while entries.bytes > weight.limit { + let (_, gone) = entries.held.pop_lru().expect("a cache over its weight holds something"); + entries.bytes -= gone.bytes; + } } /// Drop what is held under `key`: for a copy of rows that change, never /// for content under its name. pub fn forget(&self, key: &K) { - self.entries.lock().expect("content cache").pop(key); + let mut entries = self.entries.lock().expect("content cache"); + if let Some(gone) = entries.held.pop(key) { + entries.bytes -= gone.bytes; + } } /// Drop everything held. pub fn forget_all(&self) { - self.entries.lock().expect("content cache").clear(); + let mut entries = self.entries.lock().expect("content cache"); + entries.held.clear(); + entries.bytes = 0; } } @@ -53,9 +109,11 @@ impl ContentCache { /// copy of rows whose change names a group (a project's, a tenant's). pub fn forget_where(&self, whose: impl Fn(&K, &V) -> bool) { let mut entries = self.entries.lock().expect("content cache"); - let picked: Vec = entries.iter().filter(|(key, value)| whose(key, value)).map(|(key, _)| key.clone()).collect(); + let picked: Vec = entries.held.iter().filter(|(key, held)| whose(key, &held.value)).map(|(key, _)| key.clone()).collect(); for key in &picked { - entries.pop(key); + if let Some(gone) = entries.held.pop(key) { + entries.bytes -= gone.bytes; + } } } } @@ -78,4 +136,29 @@ mod tests { assert!(cache.get(&(b, "h1".to_string())).is_none(), "the least recently used went"); assert_eq!(cache.get(&(a, "h1".to_string())).as_deref(), Some(&1)); } + + /// A weighed cache holds as many values as fit in its bytes, however + /// many that is: the least recently used go until the rest fit, a value + /// put again under its key counts once, and one heavier than the whole + /// bound is not kept (nor is what its key held before). + #[test] + fn a_weighed_cache_holds_what_fits_in_its_bytes() { + let cache: ContentCache<&str, String> = ContentCache::weighed(10, String::len); + cache.put("a", Arc::new("aaaa".into())); + cache.put("b", Arc::new("bbbb".into())); + assert!(cache.get(&"a").is_some(), "8 bytes fit in 10, and `a` is now the most recently used"); + cache.put("c", Arc::new("cccc".into())); + assert!(cache.get(&"b").is_none(), "12 bytes do not fit: the least recently used went"); + assert!(cache.get(&"a").is_some() && cache.get(&"c").is_some()); + cache.put("c", Arc::new("cc".into())); + cache.put("d", Arc::new("dddd".into())); + assert!(cache.get(&"a").is_some() && cache.get(&"c").is_some() && cache.get(&"d").is_some(), "4 + 2 + 4: `c` weighs what it holds now"); + cache.put("a", Arc::new("a".repeat(11))); + assert!(cache.get(&"a").is_none(), "heavier than the bound: neither it nor what its key held is kept"); + cache.put("e", Arc::new("eeee".into())); + assert!(cache.get(&"c").is_some() && cache.get(&"d").is_some() && cache.get(&"e").is_some(), "the dropped `a` weighs nothing any more"); + cache.forget(&"e"); + cache.put("f", Arc::new("ffff".into())); + assert!(cache.get(&"c").is_some() && cache.get(&"d").is_some(), "what was forgotten weighs nothing any more"); + } } diff --git a/crates/weft-core/src/context.rs b/crates/weft-core/src/context.rs index a03022f6e..5ce6a44a2 100644 --- a/crates/weft-core/src/context.rs +++ b/crates/weft-core/src/context.rs @@ -2411,6 +2411,19 @@ impl StorageHandle { self.link(file, crate::storage::LinkReach::Internal).await } + /// A link the project's own containers fetch this file from (an + /// infra unit's container the body hands it to): at the address the + /// install's containers reach it by, never out through a tunnel and + /// back. On a local install a container's `127.0.0.1` is the + /// container itself, so an [`Self::internal_link`] opens nothing + /// there and this one goes through the install's internal port as + /// containers reach it; on a cloud install it is the internal link. + /// Lifetime as for [`Self::internal_link`]. A url-backed file value + /// is already a link and answers itself. + pub async fn container_link(&self, file: &crate::storage::FileHandle) -> WeftResult { + self.link(file, crate::storage::LinkReach::Container).await + } + /// A link anything outside the install fetches this file from (a /// browser, an app, an outside service), and whether the open /// internet can open it (`public`). On a cloud install it is the @@ -2964,8 +2977,9 @@ pub trait ContextHandle: Send + Sync { /// run file this also makes it survive the terminate sweep. async fn storage_keep(&self, key: &str, ttl: crate::storage::KeepTtl) -> WeftResult<()>; - /// A link to `key` for `reach` (see `StorageHandle::internal_link` - /// and `external_link`), living `ttl_secs` (`None`: the default). + /// A link to `key` for `reach` (see `StorageHandle::internal_link`, + /// `container_link` and `external_link`), living `ttl_secs` (`None`: + /// the default). async fn storage_link( &self, key: &str, @@ -3437,6 +3451,7 @@ mod value_bag_tests { self.note(format!("link {key} {ttl_secs:?} {reach:?}")); let link = match reach { crate::storage::LinkReach::Internal => crate::storage::FileLink { url: format!("http://store/{key}"), public: false }, + crate::storage::LinkReach::Container => crate::storage::FileLink { url: format!("http://containers/{key}"), public: false }, crate::storage::LinkReach::External => crate::storage::FileLink { url: format!("https://out/{key}"), public: self.public }, }; Ok(crate::storage::LinkAnswer { link, file: probe_meta(key) }) @@ -3494,6 +3509,7 @@ mod value_bag_tests { let linked = pure.storage_link("k", None, crate::storage::LinkReach::External).await.unwrap(); assert_eq!(linked.link.url, "https://out/k"); assert!(pure.storage_link("k", None, crate::storage::LinkReach::Internal).await.is_err()); + assert!(pure.storage_link("k", None, crate::storage::LinkReach::Container).await.is_err()); pure.storage_claim_uploads(&[("weft-upload:s", &crate::weft_type::WeftType::parse("File").unwrap())]).await.expect("its caller's upload"); let slot = crate::storage::slot::UploadSlotRequest { max_bytes: 1, @@ -3543,10 +3559,16 @@ mod value_bag_tests { let storage = ctx.storage(crate::storage::StorageScope::Run); let file = crate::storage::FileHandle::from_value(&stored).unwrap(); assert_eq!(storage.internal_link(&file).await.unwrap().url, "http://store/run/c1/put1"); + assert_eq!(storage.container_link(&file).await.unwrap().url, "http://containers/run/c1/put1"); storage.external_link(&crate::storage::FileHandle::key("run/c1/other")).await.unwrap(); assert_eq!( probe.calls(), - ["put Run a.txt", "link run/c1/put1 Some(7200) Internal", "link run/c1/other None External"] + [ + "put Run a.txt", + "link run/c1/put1 Some(7200) Internal", + "link run/c1/put1 Some(7200) Container", + "link run/c1/other None External" + ] ); } diff --git a/crates/weft-core/src/context/pure.rs b/crates/weft-core/src/context/pure.rs index dc24bbd74..87bdfa051 100644 --- a/crates/weft-core/src/context/pure.rs +++ b/crates/weft-core/src/context/pure.rs @@ -281,8 +281,9 @@ impl ContextHandle for PureHandle { /// The link its caller fetches, which a pure node answering its /// caller (Reply) needs. Making it twice is harmless: within a window /// it is the same link, and the file's life it extends only ever - /// grows to cover a link handed out. One for node code would be - /// fetched by something beside the body, outside the run. + /// grows to cover a link handed out. One for node code or its + /// containers would be fetched by something beside the body, outside + /// the run. async fn storage_link( &self, key: &str, @@ -291,7 +292,7 @@ impl ContextHandle for PureHandle { ) -> WeftResult { match reach { crate::storage::LinkReach::External => self.inner.storage_link(key, ttl_secs, reach).await, - crate::storage::LinkReach::Internal => self.refused("link a stored file for node code"), + crate::storage::LinkReach::Internal | crate::storage::LinkReach::Container => self.refused("link a stored file for node code"), } } diff --git a/crates/weft-core/src/install.rs b/crates/weft-core/src/install.rs index 3e02a802a..18b932372 100644 --- a/crates/weft-core/src/install.rs +++ b/crates/weft-core/src/install.rs @@ -347,10 +347,54 @@ crate::wire_enum! { } } +/// The weft CLI the install workflow stored for one source tree, as the +/// install answers it (`GET /install/cli/{tree}`): where to download it, +/// a link the store signs, and the sha256 of its bytes, in hex, which the +/// caller checks what it downloaded against. +// SYNC: InstallCli <-> scripts/release-cli.sh (from_install), crates/weft-dispatcher/src/api/install_cli.rs (the route) +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InstallCli { + pub url: String, + pub sha256: String, +} + +/// Whether `tree` is a git tree id: 40 hex digits (SHA-1) or 64 (SHA-256), +/// lowercase, as git prints one. What names a stored CLI, so nothing else +/// reaches the store's keys through it. +pub fn is_tree_id(tree: &str) -> bool { + matches!(tree.len(), 40 | 64) && tree.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// Where in the install's file store the CLI built from `tree` lives, and +/// the file beside it holding its sha256, which is written last, so its +/// being there is what says the CLI is whole. +// SYNC: install_cli_keys <-> .github/workflows/install-gcp.yml (store the weft CLI in the install) +pub fn install_cli_keys(tree: &str) -> (String, String) { + (format!("install/cli/{tree}/weft"), format!("install/cli/{tree}/sha256")) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_tree_id_is_40_or_64_lowercase_hex_digits() { + assert!(is_tree_id(&"a".repeat(40))); + assert!(is_tree_id(&"0123456789abcdef".repeat(4))); + assert!(!is_tree_id(&"A".repeat(40)), "uppercase"); + assert!(!is_tree_id(&"a".repeat(41))); + assert!(!is_tree_id(&format!("{}/", "a".repeat(39))), "a path"); + assert!(!is_tree_id("")); + } + + /// The answer the deploy's lookup reads (`.url`, `.sha256`). + #[test] + fn a_stored_cli_answers_the_fields_the_lookup_reads() { + let answer = serde_json::to_value(InstallCli { url: "https://u".into(), sha256: "s".into() }).unwrap(); + assert_eq!(answer, serde_json::json!({ "url": "https://u", "sha256": "s" })); + assert_eq!(serde_json::from_value::(answer).unwrap(), InstallCli { url: "https://u".into(), sha256: "s".into() }); + } + crate::wire_enum_roundtrip_tests!(DatabaseHolder); /// Only a `503` the install gave itself is asked again; the same diff --git a/crates/weft-core/src/live.rs b/crates/weft-core/src/live.rs index c07e2e0d7..67f359261 100644 --- a/crates/weft-core/src/live.rs +++ b/crates/weft-core/src/live.rs @@ -152,6 +152,25 @@ pub struct LiveItem { pub data: Value, #[serde(default, skip_serializing_if = "Option::is_none")] pub action: Option, + /// On a secret a node hands out once: the action kind a reader presses + /// as soon as it has copied the value out (`weft infra env` writing it + /// into a file, the editor's copy button), sending the value it took as + /// the press's payload ([`LiveItem::taken_press`]), so the node retires + /// only a value its taker really holds. The node then shows the secret + /// as handed over until it makes a new one. + // SYNC: taken <-> packages/weft-graph/src/protocol.ts LiveDataItem.taken, + // packages/weft-graph/src/webview/lib/components/project/ProjectNode.svelte (takenFrom), + // catalog/postgres/database/images/credential/bootstrap.py (TAKEN_ACTION, taken) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub taken: Option, + /// The line standing where such a secret was once it was handed over: + /// its value is gone from the display, and its button (`action`) is + /// the one that makes a new one. Said by the node, never read off the + /// line's shape: a plain line with a button is just that. + // SYNC: handedOver <-> packages/weft-graph/src/protocol.ts LiveDataItem.handedOver, + // catalog/postgres/database/images/credential/bootstrap.py (password_item) + #[serde(default, rename = "handedOver", skip_serializing_if = "std::ops::Not::not")] + pub handed_over: bool, } impl LiveItem { @@ -183,7 +202,7 @@ impl LiveItem { } fn of(kind: LiveItemKind, label: impl Into, data: Value) -> Self { - Self { kind, label: label.into(), data, action: None } + Self { kind, label: label.into(), data, action: None, taken: None, handed_over: false } } /// Is this item's `data` the shape its kind draws? @@ -232,6 +251,33 @@ impl LiveItem { self.action = Some(action); self } + + /// Mark this secret as one its reader retires by pressing `kind` once + /// it holds the value ([`Self::taken`]). + pub fn taken_by(mut self, kind: impl Into) -> Self { + self.taken = Some(kind.into()); + self + } + + /// Mark this line as the one standing where a secret was handed over + /// ([`Self::handed_over`]). + pub fn as_handed_over(mut self) -> Self { + self.handed_over = true; + self + } + + /// What a reader that copied this item's value out presses to say so, + /// with the value as its proof, or `None` for an item nobody retires + /// (only a secret is ever retired this way). + // SYNC: taken_press <-> packages/weft-graph/src/webview/lib/components/project/ProjectNode.svelte (the secret's copy button), + // catalog/postgres/database/images/credential/bootstrap.py (TAKEN_ACTION) + pub fn taken_press(&self) -> Option { + if self.kind != LiveItemKind::Secret { + return None; + } + let kind = self.taken.clone()?; + Some(LivePress { kind, payload: serde_json::json!({ "value": self.data }) }) + } } /// A button on one item of an infra node's display. `action_kind` is @@ -370,6 +416,30 @@ mod live_wire_tests { ); } + /// A secret handed out once names what its reader presses, and the + /// press carries the value taken; an item with none is never pressed. + #[test] + fn a_secret_handed_out_once_is_retired_with_the_value_taken() { + let sent = serde_json::json!({ + "items": [{ "type": "secret", "label": "Password", "data": "pw", "taken": "password_taken" }] + }); + let feed = LiveFeed::from_answer(&sent).expect("the envelope is right"); + assert_eq!(feed.items[0], LiveItem::secret("Password", "pw").taken_by("password_taken")); + assert_eq!(serde_json::to_value(&feed).unwrap(), sent); + assert_eq!( + feed.items[0].taken_press(), + Some(LivePress { kind: "password_taken".into(), payload: serde_json::json!({ "value": "pw" }) }) + ); + assert_eq!(LiveItem::secret("API key", "k").taken_press(), None); + let mut text = LiveItem::text("Password", "pw"); + text.taken = Some("password_taken".into()); + assert_eq!(text.taken_press(), None, "only a secret is retired by its reader"); + let gone = LiveItem::text("Password", "handed over").as_handed_over(); + assert_eq!(serde_json::to_value(&gone).unwrap()["handedOver"], true); + assert_eq!(serde_json::from_value::(serde_json::to_value(&gone).unwrap()).unwrap(), gone); + assert!(serde_json::to_value(LiveItem::text("Phone", "paired")).unwrap().get("handedOver").is_none()); + } + #[test] fn a_feed_reads_back_what_a_container_sent() { let sent = serde_json::json!({ diff --git a/crates/weft-core/src/node_test.rs b/crates/weft-core/src/node_test.rs index 852d41f08..a3b19633b 100644 --- a/crates/weft-core/src/node_test.rs +++ b/crates/weft-core/src/node_test.rs @@ -3080,7 +3080,8 @@ impl ContextHandle for TestHandle { } /// A link to a file the fake stores: one straight to the fake - /// store for node code, one under [`FAKE_LINK_BASE`] for the outside, + /// store for node code, one under [`FAKE_CONTAINER_BASE`] for the + /// project's containers, one under [`FAKE_LINK_BASE`] for the outside, /// public only once the test said so (`FakeRig::public_links`). A /// lifetime past seven days is refused, as the real store refuses it. async fn storage_link( @@ -3101,6 +3102,9 @@ impl ContextHandle for TestHandle { .map_err(WeftError::NodeFailed)?; let link = match reach { crate::storage::LinkReach::Internal => crate::storage::FileLink { url: format!("{FAKE_STORE_BASE}/{key}?ttl={ttl}"), public: false }, + crate::storage::LinkReach::Container => { + crate::storage::FileLink { url: format!("{FAKE_CONTAINER_BASE}/files/{key}?ttl={ttl}"), public: false } + } crate::storage::LinkReach::External => crate::storage::FileLink { url: format!("{FAKE_LINK_BASE}/files/{key}?ttl={ttl}"), public: self.state.public_links.load(Ordering::SeqCst), @@ -3360,6 +3364,10 @@ pub const FAKE_LINK_BASE: &str = "https://fake-weft.test"; /// (`StorageHandle::internal_link`). pub const FAKE_STORE_BASE: &str = "http://fake-store.test"; +/// The base the fake rig's links for the project's containers go under +/// (`StorageHandle::container_link`). +pub const FAKE_CONTAINER_BASE: &str = "http://fake-containers.test"; + /// The firing id every [`HandleRole::Harness`] handle runs as, keeping /// the test code's own waits out of the watchdog's picture. pub const NODE_TEST_HARNESS_ID: &str = "node-test-harness"; diff --git a/crates/weft-core/src/storage/mod.rs b/crates/weft-core/src/storage/mod.rs index 3a74f1892..3846d023d 100644 --- a/crates/weft-core/src/storage/mod.rs +++ b/crates/weft-core/src/storage/mod.rs @@ -472,8 +472,8 @@ fn first_file_version() -> u64 { /// concrete `__weft___` marker (see `to_value`). The ONLY thing /// that flows on edges / into the journal for a stored file. Carries /// NO url: byte access always goes through an authenticated fetch (or -/// an expiring link a node deliberately asks for, `internal_link` / -/// `external_link`). The +/// an expiring link a node deliberately asks for, `internal_link`, +/// `container_link` or `external_link`). The /// concrete kind (Image/Video/Audio/Blob) is derived from `mime_type`. // SYNC: StoredFile <-> packages/weft-graph/src/protocol.ts FileValueWire // (per-kind markers __weft_image__/video/audio/blob, parseFileValue) @@ -776,8 +776,8 @@ pub struct IdentityLookupResponse { /// A link to a stored file: where to fetch it, and whether the open /// internet can (a provider handed a link it cannot open gets the bytes -/// in its request instead). What `StorageHandle::internal_link` and -/// `external_link` answer. +/// in its request instead). What `StorageHandle::internal_link`, +/// `container_link` and `external_link` answer. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FileLink { pub url: String, @@ -799,6 +799,13 @@ pub enum LinkReach { /// Node code inside the install: straight to storage, at the address /// a worker reaches it by, never through a tunnel. Internal, + /// A project's own container (an infra unit's): at the address the + /// install's containers reach it by, never through a tunnel. On a + /// local install that is the install's internal port as a container + /// reaches it, through the files door there (a container's own + /// `127.0.0.1` is itself); on an install whose store every container + /// reaches at the workers' address (a cloud one), the `Internal` link. + Container, /// Anything outside the install's containers (a browser, an app, an /// outside service): the project's visitor address, else weft's /// tunnel, else the install's own address (`public: false`); on a @@ -2320,6 +2327,10 @@ mod tests { // No lifetime named: the value's own default is asked for. let bare: LinkRequest = serde_json::from_value(json!({"key": "k", "reach": "internal"})).unwrap(); assert_eq!((bare.ttl_secs, bare.reach), (None, LinkReach::Internal)); + let container = LinkRequest { key: "k".into(), ttl_secs: None, reach: LinkReach::Container }; + let v = serde_json::to_value(&container).unwrap(); + assert_eq!(v, json!({"key": "k", "reach": "container"})); + assert_eq!(serde_json::from_value::(v).unwrap().reach, LinkReach::Container); assert_eq!( serde_json::to_value(FileLink { url: "https://x/f".into(), public: false }).unwrap(), json!({"url": "https://x/f", "public": false}) diff --git a/crates/weft-dispatcher/src/api/install_cli.rs b/crates/weft-dispatcher/src/api/install_cli.rs new file mode 100644 index 000000000..85bfed2b6 --- /dev/null +++ b/crates/weft-dispatcher/src/api/install_cli.rs @@ -0,0 +1,61 @@ +//! `GET /install/cli/{tree}`: a link to the weft CLI built from the +//! source tree `tree`, as the install stores it, with its sha256. +//! +//! The install workflow stores the CLI it ran with for its own source +//! (built there, or downloaded from weft's release) beside the install's +//! files, so a project's deploy workflow downloads it instead of +//! compiling it, which matters for a fork, whose CLI exists nowhere else. +//! The broker owns the bucket and signs the link as the install's own +//! account (`admin_install_cli`); this route only names the tree and +//! passes the answer on. It is asked once per deploy and keeps nothing. + +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::Json; +use weft_core::install::{is_tree_id, InstallCli}; + +use crate::authenticator::CallerTenant; +use crate::state::DispatcherState; + +/// `GET /install/cli/{tree}`: 404 when the install stores no CLI for that +/// tree (the caller then looks elsewhere or builds one). +// SYNC: the route <-> scripts/release-cli.sh (from_install), crates/weft-core/src/install.rs (InstallCli) +pub async fn for_tree( + State(state): State, + _caller: CallerTenant, + Path(tree): Path, +) -> Result, (StatusCode, String)> { + if !is_tree_id(&tree) { + return Err((StatusCode::BAD_REQUEST, format!("'{tree}' is not a git tree id (40 or 64 lowercase hex digits)"))); + } + let asked = async { + state + .broker + .request(reqwest::Method::GET, &format!("/v1/storage/admin/install-cli/{tree}")) + .await? + .timeout(crate::storage::BROKER_STORAGE_CALL_TIMEOUT) + .send() + .await + .map_err(anyhow::Error::from) + }; + let response = asked.await.map_err(|e| { + let status = match crate::storage::storage_unreachable(&e) { + true => StatusCode::BAD_GATEWAY, + false => StatusCode::INTERNAL_SERVER_ERROR, + }; + (status, format!("ask the install's storage for the CLI built from {tree}: {e:#}")) + })?; + match response.status() { + reqwest::StatusCode::OK => response.json::().await.map(Json).map_err(|e| { + (StatusCode::BAD_GATEWAY, format!("the install's storage answered the CLI built from {tree} with something unreadable: {e}")) + }), + reqwest::StatusCode::NOT_FOUND => Err(( + StatusCode::NOT_FOUND, + format!("this install stores no weft CLI built from the source tree {tree}; its install workflow stores the one it ran with"), + )), + status => { + let body = response.text().await.unwrap_or_default(); + Err((StatusCode::BAD_GATEWAY, format!("the install's storage answered {status} for the CLI built from {tree}: {body}"))) + } + } +} diff --git a/crates/weft-dispatcher/src/api/mod.rs b/crates/weft-dispatcher/src/api/mod.rs index 1cf271519..89a9fc79d 100644 --- a/crates/weft-dispatcher/src/api/mod.rs +++ b/crates/weft-dispatcher/src/api/mod.rs @@ -208,6 +208,7 @@ impl HoldQuery { pub mod project; pub mod domains; +pub mod install_cli; // `pub` (not `pub(crate)`) like `project` above: `cancel_terminal_events` // is exercised by the db-test rig in `tests/db/db_tags.rs` against a real // Postgres. @@ -247,6 +248,9 @@ mod public_page; fn core_routes(state: DispatcherState) -> Router { Router::new() .route("/install", get(project::install)) + // The weft CLI the install stores for one source tree, which a + // project's deploy workflow downloads instead of compiling. + .route("/install/cli/{tree}", get(install_cli::for_tree)) // Whether the database held the install back lately (`weft status`). .route("/install/database", get(project::install_database)) .route("/install/domains", get(domains::list).post(domains::add)) @@ -509,13 +513,7 @@ fn outside_caller_routes(state: DispatcherState) -> Router { "/files/upload/{verb}", post(crate::files_door::upload).layer(DefaultBodyLimit::max(PUBLIC_FIRE_BODY_LIMIT)), ) - // The files door: a link to a stored file that goes through the - // install, passed on to the store unchanged; the link's signature - // is the credential (`files_door::forward`). The bytes stream - // through, so the body limit is the door's own: a PUT says its - // length, at most what a store takes in one piece. - // SYNC: the files door <-> crates/weft-platform-traits/src/object_store.rs FILES_DOOR - .route("/files/{*path}", any(crate::files_door::forward).layer(DefaultBodyLimit::disable())) + .route(FILES_DOOR_PATH, files_door()) // The root page, so a person checking the address sees // something deliberate. .route("/", get(public_page::index)) @@ -597,6 +595,29 @@ fn permissive_cors() -> CorsLayer { .max_age(std::time::Duration::from_secs(24 * 3600)) } +/// Where the files door answers. +// SYNC: the files door <-> crates/weft-platform-traits/src/object_store.rs FILES_DOOR +const FILES_DOOR_PATH: &str = "/files/{*path}"; + +/// The files door: a link to a stored file that goes through the install, +/// passed on to the store unchanged; the link's signature is the +/// credential (`files_door::forward`). The bytes stream through, so the +/// body limit is the door's own: a PUT says its length, at most what a +/// store takes in one piece. +fn files_door() -> axum::routing::MethodRouter { + any(crate::files_door::forward).layer(DefaultBodyLimit::disable()) +} + +/// The files door alone, for the port a local install's containers reach +/// the machine on (its internal port): a project's own container opens a +/// file through the link made for it (`LinkReach::Container`) there, +/// since inside a container `127.0.0.1` is the container itself. Beside +/// the roles' routes and outside their guard: the link's signature is the +/// credential, as on every other address the door answers. +pub fn container_files_router(state: DispatcherState) -> Router { + Router::new().route(FILES_DOOR_PATH, files_door()).with_state(state) +} + /// Build the dispatcher router, bound to `state`: the management surface /// and the outside doors (see [`core_routes`]). pub fn router(state: DispatcherState) -> Router { diff --git a/crates/weft-dispatcher/src/app.rs b/crates/weft-dispatcher/src/app.rs index 0550cb30a..8e3696baa 100644 --- a/crates/weft-dispatcher/src/app.rs +++ b/crates/weft-dispatcher/src/app.rs @@ -217,6 +217,7 @@ pub async fn build_state(settings: DispatcherSettings<'_>, defaults: Defaults) - } info!("dispatcher replica: {replica}"); let store = Arc::new(PostgresStore::from_pool(pool.clone())); + let programs = store.programs().clone(); let projects: crate::ProjectStore = Arc::new(crate::PostgresProjectStore::new(pool.clone())); let activations: crate::activation_store::ActivationStore = Arc::new(crate::activation_store::PostgresActivationStore::new(pool.clone())); @@ -233,7 +234,7 @@ pub async fn build_state(settings: DispatcherSettings<'_>, defaults: Defaults) - images, pool: pool.clone(), prunes: Default::default(), - blobs: crate::build::blob_cache::BlobCache::in_temp_dir(), + blobs: crate::build::blob_cache::BlobCache::in_temp_dir(config.roles.of(CoreRole::Dispatcher).files_take_memory()), }); // Every peer this client talks to (the broker and listener roles, an // infra unit's `/live` and `/action`, a worker) answers in place; a @@ -301,7 +302,7 @@ pub async fn build_state(settings: DispatcherSettings<'_>, defaults: Defaults) - weft_broker_client::TokenSource::role(tokens, replica.clone(), CoreRole::Dispatcher), ), http, - programs: Arc::new(weft_core::content_cache::ContentCache::new(64)), + programs, held, project_ports: match &config.platform { weft_platform_traits::config::PlatformConfig::Local(local) => { diff --git a/crates/weft-dispatcher/src/build/blob_cache.rs b/crates/weft-dispatcher/src/build/blob_cache.rs index 7b29c9480..ec875054b 100644 --- a/crates/weft-dispatcher/src/build/blob_cache.rs +++ b/crates/weft-dispatcher/src/build/blob_cache.rs @@ -17,8 +17,15 @@ use std::time::{Duration, SystemTime}; use anyhow::{Context, Result}; -/// The default ceiling on the cache's bytes. +/// The ceiling on the cache's bytes on a disk of the replica's own. pub const DEFAULT_CAP_BYTES: u64 = 1 << 30; +/// The ceiling where the replica's files take its memory +/// (`Placement::files_take_memory`): a cache of [`DEFAULT_CAP_BYTES`] there +/// would outgrow the whole replica, which runs out of memory on a build +/// like any other allocation. +pub const IN_MEMORY_CAP_BYTES: u64 = 64 << 20; +// At most a quarter of the least memory a role is given (512Mi). +const _: () = assert!(IN_MEMORY_CAP_BYTES * 4 <= 512 << 20); #[derive(Debug, Clone)] pub struct BlobCache { @@ -31,9 +38,12 @@ impl BlobCache { Self { dir, cap_bytes } } - /// Next to the build working directories (the system temp dir). - pub fn in_temp_dir() -> Self { - Self::new(std::env::temp_dir().join("weft-blob-cache"), DEFAULT_CAP_BYTES) + /// Next to the build working directories (the system temp dir), sized + /// for where the replica runs: [`IN_MEMORY_CAP_BYTES`] when its files + /// take its memory, else [`DEFAULT_CAP_BYTES`]. + pub fn in_temp_dir(files_take_memory: bool) -> Self { + let cap = if files_take_memory { IN_MEMORY_CAP_BYTES } else { DEFAULT_CAP_BYTES }; + Self::new(std::env::temp_dir().join("weft-blob-cache"), cap) } fn tenant_dir(&self, tenant: &str) -> Result { @@ -256,6 +266,14 @@ mod tests { assert!(got(&keep, "t", "c").is_some()); } + /// A replica whose files take its memory keeps a cache a small share of + /// any role's memory, never the gigabyte a disk of its own takes. + #[test] + fn a_replica_whose_files_take_its_memory_keeps_a_small_cache() { + assert_eq!(BlobCache::in_temp_dir(true).largest_kept(), IN_MEMORY_CAP_BYTES / 10); + assert_eq!(BlobCache::in_temp_dir(false).largest_kept(), DEFAULT_CAP_BYTES / 10); + } + /// A blob over a tenth of the cap is never kept, so one big file /// cannot flush everything else the cache holds. #[test] diff --git a/crates/weft-dispatcher/src/build/mod.rs b/crates/weft-dispatcher/src/build/mod.rs index a6837be3b..520410808 100644 --- a/crates/weft-dispatcher/src/build/mod.rs +++ b/crates/weft-dispatcher/src/build/mod.rs @@ -361,14 +361,9 @@ impl VersionBuilder { if !seen.insert(image.image_ref.clone()) { continue; } - // A worker is only there once a build of it succeeded on the - // ledger, with the record of what it left out when its build - // reports one: one a failed read left in the registry, or one - // nothing here built, is built again, never run with its gaps - // unknown (registration reads that record, `ledger::left_out`). let recorded = image.kind == weft_compiler::build_plan::ImageKind::Worker; let there = self.images.image_exists(&image.image_ref).await? - && (!recorded || ledger::succeeded(&self.pool, &image.image_ref).await?); + && (!recorded || self.built_whole(&image.image_ref).await?); if !there { stale.push(image.clone()); } @@ -395,6 +390,30 @@ impl VersionBuilder { answered.await.map_err(|_| anyhow!("the start of the project's images ended without an answer"))? } + /// Whether the worker image `image_ref`, which the store holds, may run + /// as it is. A worker is only there once a build of it succeeded on the + /// ledger, with the record of what it left out when its build reports + /// one: one a failed read left in the store, or one nothing here built, + /// is built again, never run with its gaps unknown (registration reads + /// that record, `ledger::left_out`). The one exception is the standard + /// worker, which the install made ready as it was set up (`weft + /// build-images` on a cloud, `weft daemon start` on a machine), each of + /// which refuses one that left a package out: found with no build on + /// the ledger, it is recorded as built (`ledger::note_shared`), so the + /// first stock project after an update runs on it instead of building + /// it again. A build of it the ledger has on record (one that failed, + /// or one still running) is left as it stands. + async fn built_whole(&self, image_ref: &str) -> Result { + if ledger::succeeded(&self.pool, image_ref).await? { + return Ok(true); + } + if image_ref != prune::standard_worker_ref(self.images.as_ref())? { + return Ok(false); + } + ledger::note_shared(&self.pool, image_ref, crate::lease::now_unix()).await?; + ledger::succeeded(&self.pool, image_ref).await + } + /// [`Self::start_images`]' starts and the version written down as /// waiting on them, on the task the request waits on. #[allow(clippy::too_many_arguments)] diff --git a/crates/weft-dispatcher/src/build/prune.rs b/crates/weft-dispatcher/src/build/prune.rs index a15d59b5a..26c9903ad 100644 --- a/crates/weft-dispatcher/src/build/prune.rs +++ b/crates/weft-dispatcher/src/build/prune.rs @@ -268,6 +268,12 @@ fn standard_worker_hash(images: &dyn weft_platform_traits::ImageBuilder) -> Resu weft_compiler::build::stock_binary_hash(&images.worker_packaging()).map_err(|e| anyhow::anyhow!("the standard worker's hash: {e}")) } +/// Where this weft's standard worker is in `images`' store: the image +/// every project whose nodes are all the standard library's builds to. +pub fn standard_worker_ref(images: &dyn weft_platform_traits::ImageBuilder) -> Result { + Ok(images.image_ref(&weft_compiler::build::worker_image_tag(&standard_worker_hash(images)?))) +} + /// How many of a project's earlier builds stay beside the current one, /// so going back one edit does not recompile. pub const PREVIOUS_BUILDS_KEPT: usize = 1; @@ -410,7 +416,7 @@ async fn reclaim(state: &crate::state::DispatcherState, projects: &BTreeSet>>; +/// The programs runs were started on, as a search reads them: through the +/// process's one cache of programs (`crate::state::Programs`), and the +/// ones there is none to spell their nodes by, kept for the whole search +/// so each is looked for once. +struct Programs<'a> { + read: &'a crate::state::Programs, + none: HashSet<(uuid::Uuid, String)>, +} /// Build the entry of every run queued, then build again every entry at /// an older [`FORMAT`], a round of [`ROUND`] at a time, and answer how /// many. -pub async fn index_queued(pool: &db::Pool) -> anyhow::Result { - let mut programs = Programs::new(); +pub async fn index_queued(pool: &db::Pool, programs: &crate::state::Programs) -> anyhow::Result { + let mut programs = Programs { read: programs, none: HashSet::new() }; let mut indexed = 0; for pending in Pending::BOTH { loop { @@ -199,7 +203,7 @@ pub async fn index_queued(pool: &db::Pool) -> anyhow::Result { /// record read a page at a time outside any transaction and each entry /// written on its own ([`Pending::write`]); answers how many were read. /// Everything runs on the round's one connection. -async fn round(pool: &db::Pool, programs: &mut Programs, pending: Pending) -> anyhow::Result { +async fn round(pool: &db::Pool, programs: &mut Programs<'_>, pending: Pending) -> anyhow::Result { let conn = pool.get().await?; let ids = pending.waiting(&conn).await?; for run_id in &ids { @@ -246,7 +250,7 @@ async fn round(pool: &db::Pool, programs: &mut Programs, pending: Pending) -> an /// named, or no longer reading), said in the log. async fn started_on( db: &impl Client, - programs: &mut Programs, + programs: &mut Programs<'_>, run_id: RunId, birth: Option<&RunEvent>, ) -> anyhow::Result>> { @@ -256,8 +260,11 @@ async fn started_on( }; let Some(hash) = cut_from.as_ref().or(definition_hash.as_ref()) else { return Ok(None) }; let key = (*project_id, hash.clone()); - if let Some(program) = programs.get(&key) { - return Ok(program.clone()); + if programs.none.contains(&key) { + return Ok(None); + } + if let Some(program) = programs.read.get(&key) { + return Ok(Some(program)); } let program = match crate::project_store::program_in(db, *project_id, hash).await { Ok(Some(program)) => Some(Arc::new(program)), @@ -271,7 +278,12 @@ async fn started_on( } Err(e) => return Err(e), }; - programs.insert(key, program.clone()); + match &program { + Some(program) => programs.read.put(key, program.clone()), + None => { + programs.none.insert(key); + } + } Ok(program) } diff --git a/crates/weft-dispatcher/src/state.rs b/crates/weft-dispatcher/src/state.rs index 026276b7d..f84c6f040 100644 --- a/crates/weft-dispatcher/src/state.rs +++ b/crates/weft-dispatcher/src/state.rs @@ -140,8 +140,14 @@ pub struct DispatcherState { /// Programs already read and parsed, by project and definition hash /// (see [`DispatcherState::program`]): a definition never changes /// under its hash, so a busy route stops reading and parsing its - /// program on every call. - pub programs: Arc>, + /// program on every call. Bounded by what they weigh + /// ([`PROGRAMS_JSON_BYTES`]), never by how many: an entry is anything + /// from a trigger's cut of a few kilobytes to a whole program of + /// megabytes (every build registers the program under a new hash, and + /// every run reads its own cut), so a count bounds nothing. The store's + /// own (`PostgresStore::programs`), which a search's pass reads through + /// too, so the bound is the whole process's. + pub programs: Programs, /// The rows every live call reads, held in memory and read again when /// they change (`crate::held`). pub held: Arc, @@ -160,6 +166,42 @@ pub struct DispatcherState { pub aside: weft_task_store::aside::StepAside, } +/// Parsed programs by project and definition hash, bounded by what they +/// weigh ([`programs_cache`]). +pub type Programs = Arc>; + +/// An empty [`Programs`], weighed by [`program_json_bytes`] and bounded by +/// [`PROGRAMS_JSON_BYTES`]. +pub fn programs_cache() -> Programs { + Arc::new(weft_core::content_cache::ContentCache::weighed(PROGRAMS_JSON_BYTES, program_json_bytes)) +} + +/// How many bytes of JSON the programs a dispatcher keeps parsed +/// ([`DispatcherState::programs`]) may weigh together +/// ([`program_json_bytes`]). A parsed program holds about three times its +/// JSON (`tests/resync_memory.rs` measures it), so they hold about 90 MiB +/// at most, inside the 512Mi a dispatcher is never given less than. +pub const PROGRAMS_JSON_BYTES: usize = 32 * 1024 * 1024; + +/// What `program` weighs in [`DispatcherState::programs`]: the length of +/// its JSON, counted as it is written out, nothing kept. Run once per +/// program read from the store, never on a cache hit. +pub fn program_json_bytes(program: &weft_core::ProjectDefinition) -> usize { + struct Counted(usize); + impl std::io::Write for Counted { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 += bytes.len(); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut counted = Counted(0); + serde_json::to_writer(&mut counted, program).expect("a program read from its JSON writes back as JSON"); + counted.0 +} + /// The base for outside URLs of one answer /// ([`DispatcherState::external_base`]): read the first time a path is put /// on it, then kept for the rest of that answer only, so one answer hands diff --git a/crates/weft-dispatcher/src/store/postgres.rs b/crates/weft-dispatcher/src/store/postgres.rs index 24a4c3a93..ba757841d 100644 --- a/crates/weft-dispatcher/src/store/postgres.rs +++ b/crates/weft-dispatcher/src/store/postgres.rs @@ -25,6 +25,12 @@ use weft_core::program::{RunPage, RunSummary, RunStatus}; /// signals, an answer writes its run), so one value implements all four. pub struct PostgresStore { pool: Pool, + /// The programs read from the database, parsed, by project and hash: + /// the process's one copy of each, which the dispatcher reads through + /// (`DispatcherState::programs`) and a search's pass too + /// (`crate::search_index::index_queued`), so their bound + /// (`crate::state::PROGRAMS_JSON_BYTES`) is the process's. + programs: crate::state::Programs, } /// The `run.phase` column as a phase. The column is NOT NULL and only @@ -131,7 +137,12 @@ impl PostgresStore { /// included) is applied once, before construction, by the boot's one /// schema pass. pub fn from_pool(pool: Pool) -> Self { - Self { pool } + Self { pool, programs: crate::state::programs_cache() } + } + + /// The process's one cache of parsed programs ([`Self::programs`]). + pub fn programs(&self) -> &crate::state::Programs { + &self.programs } /// Expose the inner pool for sibling modules (lease manager, @@ -1450,7 +1461,7 @@ impl RunStore for PostgresStore { // once it ended, here, by the first listing that reads one // (`crate::search_index`). if query.node.is_some() || query.search.is_some() { - crate::search_index::index_queued(&self.pool).await?; + crate::search_index::index_queued(&self.pool, &self.programs).await?; } // SYNC: list_runs (status clause) <-> crates/weft-core/src/program.rs RunStatus let where_clause = "r.tenant_id = $1 \ diff --git a/crates/weft-dispatcher/tests/db/db_build.rs b/crates/weft-dispatcher/tests/db/db_build.rs index 46134067b..2c77164e8 100644 --- a/crates/weft-dispatcher/tests/db/db_build.rs +++ b/crates/weft-dispatcher/tests/db/db_build.rs @@ -35,7 +35,7 @@ fn builder_on(pool: &Pool, images: Arc) -> VersionBuilder { images, pool: pool.clone(), prunes: Default::default(), - blobs: weft_dispatcher::build::blob_cache::BlobCache::in_temp_dir(), + blobs: weft_dispatcher::build::blob_cache::BlobCache::in_temp_dir(false), } } @@ -204,6 +204,30 @@ async fn present_images_are_skipped_without_the_gate(pool: Pool) { assert_eq!(gate.begun.load(Ordering::SeqCst), 0); } +/// The standard worker the install made ready as it was set up is in the +/// store with no build of it on the ledger: the first stock project runs +/// on it, recorded as built, instead of building it again. Any other +/// worker found that way is built again. +#[weft_db_macros::test] +async fn the_ready_standard_worker_is_used_without_a_build(pool: Pool) { + setup(&pool).await; + let fake = Arc::new(FakeImageBuilder::new()); + let standard = weft_dispatcher::build::prune::standard_worker_ref(fake.as_ref()).unwrap(); + let other = "fake.registry/weft-worker:unrecorded"; + fake.set_image_exists(&standard); + fake.set_image_exists(other); + let builder = builder(&pool, fake.clone()); + let gate = gate(); + let asked = ask(&builder, &pool, uuid::Uuid::new_v4(), "standard", &[image(ImageKind::Worker, &standard)], gate.clone()).await.unwrap(); + assert!(matches!(asked, VersionBuild::Ready { .. }), "the standard worker is there"); + assert!(fake.starts().is_empty()); + assert_eq!(gate.begun.load(Ordering::SeqCst), 0); + assert_eq!(status_of(&pool, &standard).await, "succeeded", "recorded as built"); + + waiting_on(ask(&builder, &pool, uuid::Uuid::new_v4(), "unrecorded", &[image(ImageKind::Worker, other)], gate).await.unwrap()); + assert_eq!(fake.starts().len(), 1, "a worker nothing here built is built again"); +} + /// A stale image is started once, and the request answers with /// the version waiting on it while it still runs, never waiting for its /// end; the build loop's look records the end later. diff --git a/crates/weft-dispatcher/tests/db/db_maintenance.rs b/crates/weft-dispatcher/tests/db/db_maintenance.rs index 19ec9b81a..b7287f8f3 100644 --- a/crates/weft-dispatcher/tests/db/db_maintenance.rs +++ b/crates/weft-dispatcher/tests/db/db_maintenance.rs @@ -285,7 +285,7 @@ async fn the_lift_rebuilds_from_the_version_the_running_build_came_from(pool: Po images: fake.clone(), pool: pool.clone(), prunes: Default::default(), - blobs: weft_dispatcher::build::blob_cache::BlobCache::in_temp_dir(), + blobs: weft_dispatcher::build::blob_cache::BlobCache::in_temp_dir(false), }; let underway = match builder .start_images(version, &images, &Staging::new(()), id, TENANT, Arc::new(NoGate), ImageHold::new(&pool, id)) diff --git a/crates/weft-dispatcher/tests/resync_memory.rs b/crates/weft-dispatcher/tests/resync_memory.rs new file mode 100644 index 000000000..559a2e8fa --- /dev/null +++ b/crates/weft-dispatcher/tests/resync_memory.rs @@ -0,0 +1,284 @@ +//! What the dispatcher holds in memory while it brings back the shared +//! triggers of a large program (a resync, a maintenance lift), measured +//! with an allocator that counts live bytes and their peak. +//! +//! A measurement, not a check: ignored, and run by hand. The counter is +//! the whole process's, which is why it is the only test of its binary: +//! +//! ```text +//! cargo nextest run --workspace -E 'rdeps(weft-dispatcher) & test(resync_memory)' --run-ignored only --no-capture +//! ``` +//! +//! The program is compiled from source by the real compiler and catalog: +//! `TRIGGERS` Route triggers sharing one PostgresDatabase, each answering +//! through a query, a Format and an ExecPython with a few kilobytes of +//! text each, the shape of a site's backend. Every step below is the code +//! a bring-back runs (`weft_core::run_spec`, `weft_dispatcher::store`, +//! `weft_journal::Fold`), called the way `weft_dispatcher::api::project` +//! calls it, and the program cache (`DispatcherState::programs`) filled +//! the two ways it grows: one cut per trigger fired, one program per +//! build. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use weft_core::frames::Located; +use weft_core::project::hash::ProgramIdentity; +use weft_core::ProjectDefinition; +use weft_dispatcher::store::TriggerBake; +use weft_journal::RunEvent; + +struct Counting; + +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let at = unsafe { System.alloc(layout) }; + if !at.is_null() { + grew(layout.size()); + } + at + } + + unsafe fn dealloc(&self, at: *mut u8, layout: Layout) { + unsafe { System.dealloc(at, layout) }; + LIVE.fetch_sub(layout.size(), Ordering::Relaxed); + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let at = unsafe { System.alloc_zeroed(layout) }; + if !at.is_null() { + grew(layout.size()); + } + at + } + + unsafe fn realloc(&self, at: *mut u8, layout: Layout, size: usize) -> *mut u8 { + let moved = unsafe { System.realloc(at, layout, size) }; + if !moved.is_null() { + LIVE.fetch_sub(layout.size(), Ordering::Relaxed); + grew(size); + } + moved + } +} + +fn grew(size: usize) { + let live = LIVE.fetch_add(size, Ordering::Relaxed) + size; + PEAK.fetch_max(live, Ordering::Relaxed); +} + +#[global_allocator] +static COUNTING: Counting = Counting; + +/// What one step left held (`held`) and the most it had live at once +/// (`peak`), both over what was live before it. +struct Took { + held: isize, + peak: usize, +} + +fn measure(step: impl FnOnce() -> T) -> (T, Took) { + let before = LIVE.load(Ordering::Relaxed); + PEAK.store(before, Ordering::Relaxed); + let out = step(); + let took = Took { held: LIVE.load(Ordering::Relaxed) as isize - before as isize, peak: PEAK.load(Ordering::Relaxed) - before }; + (out, took) +} + +fn mib(bytes: impl TryInto) -> String { + let bytes: i64 = bytes.try_into().unwrap_or(i64::MAX); + format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0)) +} + +fn report(what: &str, took: &Took) { + println!("{what:<62} held {:>10} peak {:>10}", mib(took.held as i64), mib(took.peak as i64)); +} + +const TRIGGERS: usize = 63; + +/// `words` words of plain text, distinct per `seed`. +fn text(seed: usize, words: usize) -> String { + (0..words).map(|w| format!("w{seed}x{w}")).collect::>().join(" ") +} + +/// A site's backend: `TRIGGERS` routes on one database, each with a few +/// kilobytes of query, template and code. +fn source() -> String { + let mut source = String::from("pg = PostgresDatabase { database: \"app\" }\n"); + for i in 0..TRIGGERS { + let query = format!("select {} from t{i} where tenant = $tenant", text(i, 150).replace(' ', ", ")); + let template = format!("{} {{{{p}}}} {}", text(i + 1000, 200), text(i + 2000, 100)); + let code = format!("return {{'out': x + ' {}'}}", text(i + 3000, 250)); + source.push_str(&format!( + "r{i} = Route {{ path: \"t{i}/items\", method: \"POST\" }}\n\ + q{i} = PostgresExecuteQuery(tenant: String) {{ query: \"{query}\", tenant: r{i}.path }}\n\ + q{i}.account = pg.access\n\ + f{i} = Format(p: String) {{ template: \"{template}\", p: r{i}.path }}\n\ + x{i} = ExecPython(x: String) -> (out: String) {{ code: \"{code}\", x: f{i}.text }}\n\ + a{i} = Reply {{ status: 200 }}\n\ + a{i}.body = x{i}.out\n" + )); + } + source +} + +fn program() -> ProjectDefinition { + let catalog = weft_catalog::FsCatalog::discover(&weft_catalog::stdlib_root().unwrap()).unwrap(); + let dir = tempfile::tempdir().unwrap(); + weft_compiler::compile_program(&source(), uuid::Uuid::new_v4(), weft_compiler::CompileFs::disk(dir.path()), &catalog) + .unwrap_or_else(|diagnostics| panic!("the fixture compiles: {diagnostics:?}")) +} + +/// The setup run of every trigger, as its record reads once it completed: +/// its birth, one capture per trigger holding that trigger's settings, its +/// ending. +fn setup_record(project: &ProjectDefinition, registered: &str, cut_hash: &str) -> Vec { + let run_id = uuid::Uuid::new_v4(); + let mut events = vec![RunEvent::RunStarted { + given: Default::default(), + run_id, + project_id: project.id, + entry_node: "r0".into(), + phase: weft_core::context::Phase::TriggerSetup, + definition_hash: Some(cut_hash.into()), + binary_hash: Some("binary".into()), + source_version: Some("source".into()), + run_kind: weft_core::run::RunKind::Project, + cut_from: Some(registered.into()), + seed: None, + instance: None, + stand_in: None, + fired_trigger: None, + instance_values: Default::default(), + picks: Default::default(), + at_unix: 1, + settings: weft_core::run_settings::RunSettings::bookkeeping(), + }]; + for node in project.nodes.iter().filter(|node| node.features.is_trigger) { + events.push(RunEvent::TriggerCaptured { + run_id, + node_id: node.id.clone(), + spec: weft_core::primitive::SignalSpec::of_kind("timer", serde_json::json!({"spec": {"kind": "after", "duration_ms": 100}})), + port_snapshot: serde_json::to_value(&node.config).unwrap(), + at_unix: 2, + }); + } + events.push(RunEvent::RunCompleted { run_id, at_unix: 3 }); + events +} + +#[test] +#[ignore = "a measurement: run by hand, alone (see the module doc)"] +fn resync_memory() { + let built = program(); + let json = serde_json::to_string(&built).unwrap(); + let triggers: Vec = built.nodes.iter().filter(|node| node.features.is_trigger).map(|node| node.id.clone()).collect(); + assert_eq!(triggers.len(), TRIGGERS); + let registered_hash = weft_core::project::hash::compute_definition_hash(&built).unwrap(); + println!("program: {} nodes, {} triggers, stored as {} of JSON", built.nodes.len(), triggers.len(), mib(json.len())); + drop(built); + + // One parse of the stored program: the unit every other number is read against. + let (registered, took) = measure(|| serde_json::from_str::(&json).unwrap()); + report("the registered program, parsed (one read)", &took); + + // The database's saved outputs, as its copy being up gives them: a fire + // reads them baked. + let saved: weft_core::infra::bake::Saved = weft_core::project::infra_places(®istered) + .into_iter() + .filter_map(|place| { + let node = registered.nodes.iter().find(|node| node.id == place.id)?; + (!node.baked_outputs.is_empty()).then(|| (place, node.baked_outputs.iter().map(|port| (port.clone(), serde_json::Value::Null)).collect())) + }) + .collect(); + let seeds: Vec = triggers.iter().map(|trigger| Located::top(trigger.as_str())).collect(); + let ((setup, _kicks), took) = measure(|| weft_core::run_spec::setup_cut(®istered, &seeds).unwrap()); + report("the setup cut of every trigger (setup_cut)", &took); + + let (_, took) = measure(|| { + for trigger in &triggers { + let fire = weft_core::run_spec::compute_trigger_fire(®istered, trigger, &serde_json::Value::Null, None, &saved); + fire.unwrap(); + } + }); + report("a fire cut per trigger, one at a time (cannot_come_on)", &took); + + let (_, took) = measure(|| { + let cuts: Vec = triggers.iter().map(|trigger| weft_core::run_spec::trigger_cut(®istered, trigger).unwrap().fingerprint).collect(); + cuts.len() + }); + report("a trigger cut per trigger, fingerprints kept (trigger_cuts_of)", &took); + + let (stored, took) = measure(|| weft_journal::record::StoredProgram::of(registered.id, &setup).unwrap()); + report("the setup cut stored for its birth (runs_program)", &took); + + // Over a day of fires, the program cache keeps the program each run + // ran, by its hash: a fire runs its trigger's cut, so every trigger's + // cut is an entry of its own. + let fill = |cache: &weft_core::content_cache::ContentCache<(uuid::Uuid, String), ProjectDefinition>| { + cache.put((registered.id, registered_hash.clone()), Arc::new(serde_json::from_str::(&json).unwrap())); + for trigger in &triggers { + let fire = weft_core::run_spec::compute_trigger_fire(®istered, trigger, &serde_json::Value::Null, None, &saved).unwrap(); + let hash = weft_core::project::hash::compute_definition_hash(&fire.program).unwrap(); + // Read back from the store the way `DispatcherState::program` does. + let read = serde_json::from_str::(&serde_json::to_string(&fire.program).unwrap()).unwrap(); + drop(fire); + cache.put((registered.id, hash), Arc::new(read)); + } + }; + // And over a day of builds, every build registers the program under a + // hash of its own, which an activation, a resync and every run of it + // reads. + let builds = |cache: &weft_core::content_cache::ContentCache<(uuid::Uuid, String), ProjectDefinition>| { + for build in 0..64 { + cache.put((registered.id, format!("build-{build}")), Arc::new(serde_json::from_str::(&json).unwrap())); + } + }; + let by_count = weft_core::content_cache::ContentCache::new(64); + let by_weight = weft_core::content_cache::ContentCache::weighed(weft_dispatcher::state::PROGRAMS_JSON_BYTES, weft_dispatcher::state::program_json_bytes); + for (bound, cache) in [("by count (64, before)", &by_count), ("by weight (after)", &by_weight)] { + let (_, took) = measure(|| fill(cache)); + report(&format!("the program cache once every trigger fired, {bound}"), &took); + cache.forget_all(); + let (_, took) = measure(|| builds(cache)); + report(&format!("the program cache after 64 builds, {bound}"), &took); + cache.forget_all(); + } + + // While anybody on the process follows the project (a setup's own wait + // does), the live view keeps a projection of every run that writes a + // row, each a fold over that run's program. + let setup = Arc::new(setup); + let (fold, took) = measure(|| weft_journal::Fold::new(uuid::Uuid::new_v4(), setup.clone()).showing()); + report("the live view's projection of the setup run", &took); + drop(fold); + let fires: Vec> = triggers + .iter() + .map(|trigger| Arc::new(weft_core::run_spec::compute_trigger_fire(®istered, trigger, &serde_json::Value::Null, None, &saved).unwrap().program)) + .collect(); + let (folds, took) = measure(|| fires.iter().map(|fire| weft_journal::Fold::new(uuid::Uuid::new_v4(), fire.clone()).showing()).collect::>()); + report("the live view's projections of one fire of each trigger", &took); + drop(folds); + drop(fires); + + // The setup's record read whole, its bake, the earlier bake read back, + // the two merged and written: what `run_trigger_setup` and + // `finish_trigger_setup` hold at once. + let identity = ProgramIdentity { definition_hash: registered_hash.clone(), binary_hash: "binary".into(), implementations: Default::default() }; + let record = setup_record(®istered, ®istered_hash, &stored.definition_hash); + let (_, took) = measure(|| { + // What `events_log` hands back: the record, decoded. + let events: Vec = record.clone(); + let bake = TriggerBake::from_events(&events, &identity).unwrap().unwrap(); + let prior_json = serde_json::to_string(&bake).unwrap(); + let prior: TriggerBake = serde_json::from_str(&prior_json).unwrap(); + let merged = prior.refreshed_by(&bake); + let written = serde_json::to_string(&merged).unwrap(); + (events.len(), written.len()) + }); + report("the setup's record, its bake, the prior, merged, written", &took); +} diff --git a/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/images/container_fetch/Dockerfile b/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/images/container_fetch/Dockerfile new file mode 100644 index 000000000..81a907112 --- /dev/null +++ b/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/images/container_fetch/Dockerfile @@ -0,0 +1,12 @@ +# The e2e rig's fetching container: a stdlib HTTP server that downloads the +# link it is posted and answers how many bytes it read. No dependencies, so +# no build step. Runs as a non-root user to satisfy a restricted container +# baseline. +FROM python:3.15-alpine + +WORKDIR /app +COPY server.py /app/server.py + +EXPOSE 8080 +USER 1000 +CMD ["python", "/app/server.py"] diff --git a/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/images/container_fetch/server.py b/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/images/container_fetch/server.py new file mode 100644 index 000000000..b8b43a950 --- /dev/null +++ b/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/images/container_fetch/server.py @@ -0,0 +1,51 @@ +"""The e2e rig's fetching container. + + - GET /health -> 200, the readiness probe target. + - POST /fetch {"url"} -> downloads the url from inside the container and + answers {"size": }, or 502 with what the download said. + +No dependencies (stdlib), so the image builds with no build step. +""" + +import json +import os +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +PORT = int(os.environ.get("PORT", "8080")) + + +class Handler(BaseHTTPRequestHandler): + def _send(self, code, body): + payload = json.dumps(body).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self): + if self.path == "/health": + self._send(200, {"status": "ok"}) + else: + self._send(404, {"error": "not found"}) + + def do_POST(self): + if self.path != "/fetch": + self._send(404, {"error": "not found"}) + return + length = int(self.headers.get("Content-Length", "0")) + url = json.loads(self.rfile.read(length))["url"] + try: + with urllib.request.urlopen(url, timeout=30) as answer: + self._send(200, {"size": len(answer.read())}) + except (urllib.error.URLError, OSError) as e: + self._send(502, {"error": f"downloading {url} from inside the container failed: {e}"}) + + def log_message(self, *args): + pass + + +if __name__ == "__main__": + ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() diff --git a/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/metadata.json b/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/metadata.json new file mode 100644 index 000000000..643a6c763 --- /dev/null +++ b/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/metadata.json @@ -0,0 +1,30 @@ +{ + "type": "ContainerFetch", + "label": "Container Fetch", + "description": "An e2e infra node: a container that downloads the file it is handed a link to and answers how many bytes it read. Used by the e2e rig to prove a project's own container opens a file's container link.", + "tags": [ + "infra", + "e2e" + ], + "icon": "Server", + "color": "#22c55e", + "inputs": [ + { + "name": "file", + "type": "Blob", + "label": "File", + "description": "The file the container downloads." + } + ], + "outputs": [ + { + "name": "size", + "type": "Number" + } + ], + "requiresInfra": true, + "images": [ + "images/container_fetch" + ], + "features": {} +} diff --git a/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/mod.rs b/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/mod.rs new file mode 100644 index 000000000..d27a45bd9 --- /dev/null +++ b/crates/weft-e2e/fixtures/container_link/nodes/container_fetch/mod.rs @@ -0,0 +1,50 @@ +//! ContainerFetch: an infra node whose container downloads a stored file +//! through the link the node hands it, so the e2e rig can prove a +//! project's own container opens a file's container link +//! (`StorageHandle::container_link`). On a local install a container's +//! `127.0.0.1` is the container itself, which is what an internal link +//! would point it at. + +use async_trait::async_trait; + +use weft::infra::{ + Container, ContainerPort, Endpoint, EndpointTarget, Expose, Image, InfraSpec, Limits, Probe, Protocol, Unit, +}; +use weft::node::NodeOutput; +use weft::storage::{FileHandle, StorageScope}; +use weft::{InfraProvisionContext, Node, NodeManifest, RunContext, ValueBag, WeftResult}; + +#[derive(NodeManifest)] +pub struct ContainerFetchNode; + +const PORT: u16 = 8080; + +#[async_trait] +impl Node for ContainerFetchNode { + async fn provision_infra(&self, _ctx: InfraProvisionContext, _input: ValueBag) -> WeftResult { + Ok(InfraSpec { + units: vec![Unit { + name: "fetcher".into(), + containers: vec![Container::new("app", Image::Local { name: "container_fetch".into() }) + .with_ports(vec![ContainerPort { name: "http".into(), port: PORT, protocol: Protocol::Tcp }]) + .with_limits(Limits { cpu: Some("0.25".into()), memory: Some("128Mi".into()) }) + .with_readiness(Probe::http("/health", PORT).with_initial_delay(2))], + ..Default::default() + }], + endpoints: vec![Endpoint { + name: "api".into(), + target: EndpointTarget::Unit { unit: "fetcher".into(), container: "app".into(), port: "http".into() }, + expose: Expose::Project, + }], + ..Default::default() + }) + } + + async fn run(&self, ctx: RunContext) -> WeftResult<()> { + let file: FileHandle = ctx.inputs.get("file")?; + let link = ctx.storage(StorageScope::Project).container_link(&file).await?; + let api = ctx.endpoint("api").await?; + let answer = api.call(weft::EndpointMethod::Post, "/fetch", Some(serde_json::json!({ "url": link.url }))).await?; + ctx.pulse_downstream(NodeOutput::new().set("size", answer["size"].clone())).await + } +} diff --git a/crates/weft-e2e/fixtures/container_link/src/main.weft b/crates/weft-e2e/fixtures/container_link/src/main.weft new file mode 100644 index 000000000..aaa617e63 --- /dev/null +++ b/crates/weft-e2e/fixtures/container_link/src/main.weft @@ -0,0 +1,12 @@ +# A project's own container downloads a stored file. ContainerFetch (a +# project-local infra node under nodes/container_fetch/) runs a tiny HTTP +# server in a container; its run asks for the file's container link and +# hands it to that server, which downloads it and answers how many bytes it +# got. The node starts with NO `file` set: the test writes one +# `@asset("assets/data.bin", Blob)` line, and asserts the count the +# container read equals the payload it wrote. + +fetch = ContainerFetch + +out = Debug +out.data = fetch.size diff --git a/crates/weft-e2e/fixtures/container_link/weft.toml b/crates/weft-e2e/fixtures/container_link/weft.toml new file mode 100644 index 000000000..68a00d86d --- /dev/null +++ b/crates/weft-e2e/fixtures/container_link/weft.toml @@ -0,0 +1,4 @@ +[package] +name = "e2e_container_link" +id = "00000000-0000-0000-0000-0000000c0417" +version = "0.1.0" diff --git a/crates/weft-e2e/tests/e2e/container_link.rs b/crates/weft-e2e/tests/e2e/container_link.rs new file mode 100644 index 000000000..9bd9ff263 --- /dev/null +++ b/crates/weft-e2e/tests/e2e/container_link.rs @@ -0,0 +1,28 @@ +//! A project's own container opens a stored file through the link made for +//! it (`container_link`). On a local install a container's `127.0.0.1` is +//! the container itself, so only a link that reaches the machine the way +//! containers do gets the bytes there; the container reads them back and +//! the count it read reaches Debug. +#![cfg(feature = "e2e")] + +use serde_json::json; +use weft_e2e::{ensure, infra, project::Project, run}; + +#[tokio::test] +async fn a_project_s_own_container_downloads_a_stored_file_by_its_link() -> anyhow::Result<()> { + let disp = ensure::up().await?; + let mut project = Project::prepare("container_link", disp).await?; + let payload = b"weft-e2e: read from inside a project's own container".to_vec(); + let assets = project.dir().join("assets"); + std::fs::create_dir_all(&assets)?; + std::fs::write(assets.join("data.bin"), &payload)?; + project.set_node_config("fetch", "file", "@asset(\"assets/data.bin\", Blob)")?; + infra::start_and_wait_running(&mut project, "fetch").await?; + + let settled = run::run_and_settle(&mut project).await?; + settled.completed()?; + settled.assert_input("out", "data", &json!(payload.len()))?; + + infra::terminate_and_wait_gone(&project, "fetch").await?; + project.finish().await +} diff --git a/crates/weft-e2e/tests/e2e/main.rs b/crates/weft-e2e/tests/e2e/main.rs index 45de2bc6b..7f5199f18 100644 --- a/crates/weft-e2e/tests/e2e/main.rs +++ b/crates/weft-e2e/tests/e2e/main.rs @@ -17,6 +17,7 @@ mod common; mod compile_what_you_touch; mod computer; mod config_media; +mod container_link; mod custom_node; mod custom_type; mod email; diff --git a/crates/weft-e2e/tests/e2e/postgres.rs b/crates/weft-e2e/tests/e2e/postgres.rs index 1d7ac784e..3edbdb7fa 100644 --- a/crates/weft-e2e/tests/e2e/postgres.rs +++ b/crates/weft-e2e/tests/e2e/postgres.rs @@ -21,6 +21,9 @@ //! - a watching query reached through the pooler keeps its one //! connection past the minute after which the database closes an //! idle session, and a write still fires it; +//! - the roles the node lists keep tenants apart under a `FORCE` +//! policy, one marked to bypass it writes past it, and a login role +//! signs in through the pooler with its own password; //! - a database named with a dash keeps its name, and one the pooler //! cannot read as written is reached through a door made from it; //! - terminating takes the connection away with the database. @@ -208,6 +211,9 @@ async fn the_password_goes_into_an_env_file_and_a_reset_gets_a_new_one() -> Resu ["infra", "env", "db", "--into", &into, "--set", "PG_PASSWORD=Password", "--set", "PG_USER=User"]; let written = project.weft(&secret).await?; anyhow::ensure!(written.contains("PG_USER="), "env should print the plain values it wrote, its lines: {:?}", names(&written)); + // Written into the file is taken: the card shows it handed over now. + let taken = project.weft_refused(&only_secret).await?; + anyhow::ensure!(taken.contains("weft infra press"), "the password env wrote should be handed over: {taken}"); let user_line = std::fs::read_to_string(&env)?; anyhow::ensure!( user_line.lines().any(|l| l.starts_with("PG_USER=") && l.len() > "PG_USER=".len()), @@ -297,6 +303,128 @@ async fn a_client_outside_weft_opens_the_database_through_its_door() -> Result<( project.finish().await } +/// The roles the next test lists: one per kind of tenant, one that signs +/// in by itself, one that writes past every policy. +const ROLES: &str = r#"[{"name": "tenant_app"}, {"name": "tenant_site", "login": true}, {"name": "tenant_admin", "bypassRowLevelSecurity": true}]"#; + +/// Whether `outcome` is the refusal of a row-level security policy. +fn refused_by_a_policy(outcome: Result) -> bool { + matches!(outcome, Err(e) if e.to_string().contains("row-level security")) +} + +/// How a program keeps tenants apart with roles the node lists: `app` +/// (the tables' owner, bound by a `FORCE` policy like everyone) `set +/// role`s to a tenant's role and writes only that tenant's rows, a role +/// marked `bypassRowLevelSecurity` writes past the policy, and a `login` +/// role signs in through the pooler with its own password from the card +/// (`app`'s does not open it), handed out once and minted again by its own +/// Reset button. Taking roles off the list restarts the database, which +/// sets up the one still listed again without failing, and takes back what +/// the list gave the others (`set role`, signing in, passing policies) +/// while keeping them and their rows. +#[tokio::test] +async fn roles_keep_tenants_apart_and_a_login_role_signs_in_by_itself() -> Result<()> { + let disp = ensure::up().await?; + let mut project = Project::prepare("postgres_db", disp).await?; + project.substitute_in_main("__E2E_BODY__", BODY)?; + project.set_node_config("db", "reachable", "true")?; + project.set_node_config("db", "roles", ROLES)?; + infra::start_and_wait_running(&mut project, "db").await?; + + let app = outside_client(&project, "db", "front.env").await?; + let owner = db::Session::connect(&app).await?; + owner + .batch_execute( + "CREATE TABLE tenants (tenant text NOT NULL); \ + ALTER TABLE tenants ENABLE ROW LEVEL SECURITY; ALTER TABLE tenants FORCE ROW LEVEL SECURITY; \ + CREATE POLICY own ON tenants USING (tenant = current_user) WITH CHECK (tenant = current_user); \ + GRANT SELECT, INSERT ON tenants TO tenant_app, tenant_site, tenant_admin", + ) + .await?; + anyhow::ensure!( + refused_by_a_policy(owner.execute("INSERT INTO tenants VALUES ('tenant_app')", &[]).await), + "a FORCE policy binds the tables' owner too" + ); + owner.batch_execute("SET ROLE tenant_app; INSERT INTO tenants VALUES ('tenant_app')").await?; + anyhow::ensure!( + refused_by_a_policy(owner.execute("INSERT INTO tenants VALUES ('tenant_site')", &[]).await), + "a tenant's role writes only its own rows" + ); + owner.batch_execute("SET ROLE tenant_admin; INSERT INTO tenants VALUES ('tenant_site'); RESET ROLE").await?; + + // The login role, with its own password from the card. + let env = project.dir().join("site.env"); + let into = env.to_string_lossy().to_string(); + project.weft(&["infra", "env", "db", "--into", &into, "--set", "SITE_PASSWORD=tenant_site password"]).await?; + let text = std::fs::read_to_string(&env)?; + let password = text + .lines() + .find_map(|l| l.strip_prefix("SITE_PASSWORD=")) + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + .ok_or_else(|| anyhow::anyhow!("the env file does not set SITE_PASSWORD"))?; + let mut site = app.clone(); + site.user("tenant_site").password(password.as_str()); + let signed_in = db::Session::connect(&site).await?; + let seen: Vec = signed_in.scalars("SELECT tenant FROM tenants", &[]).await?; + anyhow::ensure!(seen == ["tenant_site"], "the login role sees its own rows only, got {seen:?}"); + drop(signed_in); + let mut wrong = site.clone(); + wrong.password(app.get_password().unwrap_or_default()); + anyhow::ensure!(db::Session::connect(&wrong).await.is_err(), "app's password does not sign the login role in"); + + // Handed out once, like app's: taken by the env file, then a reset + // mints a new one, which the old one no longer signs in after. + let again: [&str; 7] = ["infra", "env", "db", "--into", &into, "--set", "SITE_PASSWORD=tenant_site password"]; + let refused = project.weft_refused(&again).await?; + anyhow::ensure!(refused.contains("weft infra press db reset_password:tenant_site"), "{refused}"); + project.weft(&["infra", "press", "db", "reset_password:tenant_site"]).await?; + project.weft(&again).await?; + let fresh = std::fs::read_to_string(&env)? + .lines() + .find_map(|l| l.strip_prefix("SITE_PASSWORD=")) + .map(|v| v.trim().to_string()) + .ok_or_else(|| anyhow::anyhow!("the env file does not set SITE_PASSWORD after the reset"))?; + anyhow::ensure!(fresh != password, "the reset minted a new password"); + anyhow::ensure!(db::Session::connect(&site).await.is_err(), "the old password signs in no more"); + let password = fresh; + site.password(password.as_str()); + db::Session::connect(&site).await?; + drop(owner); + + // Off the list: the restart sets up the one still listed again, and the + // two taken off stay in the database with their rows, but lose what the + // list gave them: `app`'s `set role`, signing in, writing past policies. + project.set_node_config("db", "roles", r#"[{"name": "tenant_app"}]"#)?; + project.weft(&["infra", "upgrade"]).await?; + infra::wait_running(&project, "db").await?; + let app = outside_client(&project, "db", "front.env").await?; + let owner = db::Session::connect(&app).await?; + let gone = "FROM pg_roles WHERE rolname IN ('tenant_site', 'tenant_admin')"; + let kept: i64 = owner.scalar(&format!("SELECT count(*) {gone}"), &[]).await?; + let login: bool = owner.scalar(&format!("SELECT bool_or(rolcanlogin) {gone}"), &[]).await?; + let bypass: bool = owner.scalar(&format!("SELECT bool_or(rolbypassrls) {gone}"), &[]).await?; + anyhow::ensure!(kept == 2, "roles taken off the list stay in the database"); + anyhow::ensure!(!login && !bypass, "they sign in no more and pass the policies no more (login {login}, bypass {bypass})"); + for gone in ["tenant_site", "tenant_admin"] { + let refused = owner.batch_execute(&format!("SET ROLE {gone}")).await; + anyhow::ensure!(refused.is_err(), "the program can no longer `set role {gone}`"); + } + owner.batch_execute("SET ROLE tenant_app").await?; + owner.batch_execute("RESET ROLE; ALTER TABLE tenants NO FORCE ROW LEVEL SECURITY").await?; + let rows: i64 = owner.scalar("SELECT count(*) FROM tenants", &[]).await?; + anyhow::ensure!(rows == 2, "the rows stay, got {rows}"); + let mut site = app.clone(); + site.user("tenant_site").password(password.as_str()); + anyhow::ensure!(db::Session::connect(&site).await.is_err(), "the pooler lets a role taken off the list in no more"); + let card = project.weft(&["infra", "show", "db"]).await?; + anyhow::ensure!(!card.contains("tenant_site password"), "its card line goes: {card}"); + drop(owner); + + infra::terminate_and_wait_gone(&project, "db").await?; + project.finish().await +} + /// The next line of the ndjson answer `answer` as JSON, reading what /// arrives until one is whole (`pending` keeps what came after it), and /// failing after `within` with what was read. diff --git a/crates/weft-engine/src/storage.rs b/crates/weft-engine/src/storage.rs index ec75b4bdb..233131423 100644 --- a/crates/weft-engine/src/storage.rs +++ b/crates/weft-engine/src/storage.rs @@ -1008,6 +1008,7 @@ mod fake { // not open. let link = match reach { LinkReach::Internal => FileLink { url: format!("http://fake-bucket/runtime/{key}?ttl={ttl}"), public: false }, + LinkReach::Container => FileLink { url: format!("http://fake-containers/files/runtime/{key}?ttl={ttl}"), public: false }, LinkReach::External => FileLink { url: format!("http://fake-install/files/runtime/{key}?ttl={ttl}"), public: false }, }; Ok(LinkAnswer { link, file }) diff --git a/crates/weft-platform-local/src/docker.rs b/crates/weft-platform-local/src/docker.rs index 62e3c6fbb..c44856c0e 100644 --- a/crates/weft-platform-local/src/docker.rs +++ b/crates/weft-platform-local/src/docker.rs @@ -426,6 +426,61 @@ pub fn host_gateway_args() -> [String; 2] { ["--add-host".into(), "host.docker.internal:host-gateway".into()] } +/// The address the project's containers reach this machine at, when they +/// reach it somewhere other than loopback: what the engine resolves +/// `host-gateway` to, asked of the engine itself in a throwaway container +/// of `image` (the runtime's own image, which the install already has, so +/// nothing is pulled), since a daemon setting (`host-gateway-ip`) can move +/// it off the default bridge. Asked on every start of the runtime, so a +/// change to Docker's network is picked up by the next start. +/// +/// Where it is an address of this machine, the engine runs on this +/// machine's own kernel (Linux) and a container's call comes in on that +/// bridge: weft's internal port listens there too, and only there beside +/// loopback, so the machine's network never reaches it. Where it is not, +/// the engine runs in a machine of its own (Docker Desktop and the like), +/// which brings a container's call to this machine's loopback: `None`. +/// Refused, naming the probe, when the engine names no address at all. +pub async fn containers_call_in_at(docker: &dyn Docker, image: &str) -> anyhow::Result> { + let [flag, host] = host_gateway_args(); + let args: Vec = + ["run", "--rm", &flag, &host, "--entrypoint", "getent", image, "hosts", "host.docker.internal"].map(String::from).into(); + let answer = docker + .exec(&args) + .await? + .ok(&args) + .map_err(|e| e.context("ask Docker which address the project's containers reach this machine at"))?; + containers_address(&answer, image, |address| std::net::TcpListener::bind((address, 0)).map(drop)) +} + +/// [`containers_call_in_at`]'s decision on what the probe printed +/// (`getent hosts`: `172.17.0.1 host.docker.internal`), with `bind` +/// trying the address (a bind succeeds only on an address this machine +/// has, and an address it does not have is the one refusal that means the +/// engine runs elsewhere; any other refusal is said). +fn containers_address( + answer: &str, + image: &str, + bind: impl Fn(std::net::IpAddr) -> std::io::Result<()>, +) -> anyhow::Result> { + let Some(address) = answer.split_whitespace().next().and_then(|word| word.parse().ok()) else { + anyhow::bail!( + "Docker named no address for this machine as its containers reach it (`host-gateway` gave {:?}), so they \ + could not reach weft's internal port; check that `docker run --rm --add-host \ + host.docker.internal:host-gateway --entrypoint getent {image} hosts host.docker.internal` prints an address", + answer.trim() + ); + }; + match bind(address) { + Ok(()) => Ok(Some(address)), + Err(e) if e.kind() == std::io::ErrorKind::AddrNotAvailable => Ok(None), + Err(e) => anyhow::bail!( + "Docker says its containers reach this machine at {address}, which is this machine's, but weft cannot listen \ + there ({e}), so they could not reach weft's internal port" + ), + } +} + /// The network every container weft starts joins, so an infra container /// reaches another by name. pub const NETWORK: &str = "weft"; @@ -517,6 +572,25 @@ pub mod fake { mod tests { use super::*; + /// The containers' address is used when this machine has it, left out + /// when the engine runs elsewhere (the address is not this machine's), + /// and refused when Docker names none or the address cannot be listened + /// on for another reason. + #[test] + fn the_containers_address_is_this_machine_s_or_none() { + let gateway: std::net::IpAddr = [172, 17, 0, 1].into(); + let answer = "172.17.0.1 host.docker.internal\n"; + assert_eq!(containers_address(answer, "img", |_| Ok(())).unwrap(), Some(gateway)); + let elsewhere = containers_address(answer, "img", |_| Err(std::io::ErrorKind::AddrNotAvailable.into())).unwrap(); + assert_eq!(elsewhere, None); + let denied = containers_address(answer, "img", |_| Err(std::io::ErrorKind::PermissionDenied.into())).unwrap_err(); + assert!(denied.to_string().contains("cannot listen there"), "{denied}"); + for none in ["", "host.docker.internal"] { + let refused = containers_address(none, "img", |_| Ok(())).unwrap_err(); + assert!(refused.to_string().contains("--entrypoint getent img hosts"), "{refused}"); + } + } + #[tokio::test] async fn a_container_already_going_counts_as_removed() { let docker = fake::FakeDocker::new(); diff --git a/crates/weft-platform-traits/src/config.rs b/crates/weft-platform-traits/src/config.rs index 39b346434..fb6c5bea0 100644 --- a/crates/weft-platform-traits/src/config.rs +++ b/crates/weft-platform-traits/src/config.rs @@ -97,7 +97,9 @@ pub struct LocalPlatform { pub data_dir: std::path::PathBuf, /// The machine's internal port as a container reaches it /// (`http://host.docker.internal:14113`): the base an infra - /// container's agent calls the broker under. + /// container's agent calls the broker under, and the one a link to a + /// stored file for a project's containers goes through + /// (`InstallConfig::container_files_base`). #[serde(rename = "containerInternalUrl")] pub container_internal_url: String, /// The image `weft-runtime` runs from (the agent beside every infra @@ -309,7 +311,12 @@ pub struct GcpPlatform { pub struct Listen { /// The public port: the management API, the public door, the relay. pub public: SocketAddr, - /// The internal port: every role's internal endpoints. + /// The internal port, on loopback: every role's internal endpoints, + /// for the processes on this machine (its workers). The runtime also + /// opens it on the address the project's containers reach this + /// machine at, when that is not loopback, which it asks Docker on + /// every start (`weft_platform_local::docker::containers_call_in_at`), + /// never on every interface. pub internal: SocketAddr, /// A port carrying only the doors outside callers use, for a tunnel /// that brings the open internet to an install whose management API @@ -657,6 +664,16 @@ impl InstallConfig { Some(base.trim_end_matches('/').to_string()) } + /// Where a project's own containers reach the install's files door + /// (`object_store::FILES_DOOR`), when they cannot reach the store at + /// the address the workers do: a local install's internal port as a + /// container reaches it, since a container's `127.0.0.1` is itself. + /// `None` on a platform without a machine, whose containers reach the + /// store where the workers do. + pub fn container_files_base(&self) -> Option { + self.machine_base(Vantage::Private) + } + /// One role's internal address as a caller at `from` reaches it; /// `None` for the dispatcher on the machine and for the holder, which /// have none. @@ -799,6 +816,15 @@ pub(crate) mod tests { c } + /// A local install's containers reach its files door at the internal + /// port as a container reaches it; a cloud install's reach the store + /// where its workers do, so it names none. + #[test] + fn a_local_install_names_where_its_containers_reach_the_files_door() { + assert_eq!(local().container_files_base().as_deref(), Some("http://host.docker.internal:14113")); + assert_eq!(gcp().container_files_base(), None); + } + /// Every serverless role that opens the database has a ceiling on its /// copies, and only those roles have one. #[test] diff --git a/crates/weft-platform-traits/src/object_store.rs b/crates/weft-platform-traits/src/object_store.rs index c4aa17af5..262efd53b 100644 --- a/crates/weft-platform-traits/src/object_store.rs +++ b/crates/weft-platform-traits/src/object_store.rs @@ -85,8 +85,9 @@ pub enum PresignAudience { /// runtime does its own I/O on, which the workers reach it at too. Install, /// Somebody who reaches the store through the install's forwarder - /// ([`FILES_DOOR`]) at `base` (`https://`, or a site that passes - /// `/weft/...` on: `https://shop.example/weft`): sign for `base`'s host, + /// ([`FILES_DOOR`]) at `base` (`https://`, a site that passes + /// `/weft/...` on: `https://shop.example/weft`, or a local install's + /// internal port as its containers reach it): sign for `base`'s host, /// which the forwarder hands the store as `X-Forwarded-Host`, and answer /// the link under `base` ([`forwarded_link`]). Forwarded { base: String }, @@ -97,7 +98,7 @@ pub enum PresignAudience { /// `//?`), so a link signed for the /// install's public address reaches the bytes with every header (`Range`, /// `If-None-Match`) intact. -// SYNC: FILES_DOOR <-> crates/weft-dispatcher/src/api/mod.rs (outside_caller_routes), packages/weft-connect/src/server/passthrough.ts PASSED_DOORS +// SYNC: FILES_DOOR <-> crates/weft-dispatcher/src/api/mod.rs (FILES_DOOR_PATH), packages/weft-connect/src/server/passthrough.ts PASSED_DOORS pub const FILES_DOOR: &str = "files"; /// The part of the bucket the forwarder passes on: the runtime files, and @@ -1379,6 +1380,11 @@ mod tests { assert!(url(PresignAudience::Forwarded { base: "https://shop.example/weft".into() }) .await .starts_with("https://shop.example/weft/files/a/b?")); + // A local install's containers, at its internal port as they + // reach it. + assert!(url(PresignAudience::Forwarded { base: "http://host.docker.internal:14113".into() }) + .await + .starts_with("http://host.docker.internal:14113/files/a/b?")); } fn link(audience: PresignAudience) -> GetLink { diff --git a/crates/weft-platform-traits/src/roles.rs b/crates/weft-platform-traits/src/roles.rs index 512d51589..6a6192c82 100644 --- a/crates/weft-platform-traits/src/roles.rs +++ b/crates/weft-platform-traits/src/roles.rs @@ -94,6 +94,17 @@ impl Placement { Self::Serverless | Self::Pool => Vantage::Private, } } + + /// Whether what a process placed here writes to its own files takes + /// its memory: a serverless or pooled copy (Cloud Run) has no disk, + /// and its file system is held in the memory the copy is given, so a + /// file it keeps counts against that limit like anything it allocates. + pub fn files_take_memory(self) -> bool { + match self { + Self::Machine => false, + Self::Serverless | Self::Pool => true, + } + } } /// Where a caller stands, which decides the address it reaches a role at: diff --git a/crates/weft-runtime/Cargo.toml b/crates/weft-runtime/Cargo.toml index de910a430..8e900f116 100644 --- a/crates/weft-runtime/Cargo.toml +++ b/crates/weft-runtime/Cargo.toml @@ -49,6 +49,9 @@ parking_lot = { workspace = true } # The process's allocator, the one every worker runs on # (`weft_engine::Jemalloc`): it hands freed memory back to the system. tikv-jemallocator = { workspace = true } +# `dup2`: the log moves the process's stderr onto each fresh file it rolls +# to (`log_file`). +libc = "0.2" weft-workspace-hack = { version = "0.1", path = "../weft-workspace-hack" } [dev-dependencies] diff --git a/crates/weft-runtime/src/lib.rs b/crates/weft-runtime/src/lib.rs index 7cb603854..6985e623c 100644 --- a/crates/weft-runtime/src/lib.rs +++ b/crates/weft-runtime/src/lib.rs @@ -308,6 +308,7 @@ pub async fn serve(config: InstallConfig, only: Option) -> anyhow::Res service_url: config.public_url.clone(), internet: config.internet_address(), own_base_url: config.own_base_url().to_string(), + container_files_base: config.container_files_base(), object_store_public_internet: config.object_store.public_internet(), slot_key: weft_core::storage::slot::SlotKey::of(&install_secret()?), line_max_age: platform::line_max_age(&config, serverless), @@ -374,6 +375,11 @@ pub async fn serve(config: InstallConfig, only: Option) -> anyhow::Res if local.listen.network.is_some() { network = Some(weft_dispatcher::api::network_router(state.clone())); } + // The port containers reach the machine on answers the files + // door too, for a link made for a project's own container + // (`LinkReach::Container`). A cloud's containers reach the + // store itself, so only here. + internal = internal.merge(weft_dispatcher::api::container_files_router(state.clone())); } let api = weft_dispatcher::api::router(state.clone()); let door = weft_dispatcher::door::router(state.clone(), api); @@ -652,6 +658,20 @@ pub async fn serve(config: InstallConfig, only: Option) -> anyhow::Res let internal_listener = tokio::net::TcpListener::bind(local.listen.internal) .await .with_context(|| format!("bind the internal port {}", local.listen.internal))?; + // The same internal port where the project's containers call + // the machine, and nowhere else: asked of Docker on this start, + // so a change to Docker's network is picked up by the next one. + let containers = weft_platform_local::docker::containers_call_in_at(&weft_platform_local::DockerCli, &local.runtime_image) + .await? + .map(|address| std::net::SocketAddr::new(address, local.listen.internal.port())); + let containers_listener = match containers { + Some(addr) => Some( + tokio::net::TcpListener::bind(addr) + .await + .with_context(|| format!("bind the internal port where the project's containers reach this machine, {addr}"))?, + ), + None => None, + }; let public_listener = tokio::net::TcpListener::bind(local.listen.public) .await .with_context(|| format!("bind the public port {}", local.listen.public))?; @@ -662,17 +682,22 @@ pub async fn serve(config: InstallConfig, only: Option) -> anyhow::Res // them can be, does it take the claims. weft_dispatcher::app::spawn_supervised("hold", weft_listener::hold::run(state, weft_listener::hold::Share::Every, std::future::pending())); } - tracing::info!(target: "weft_runtime", roles = ?hosted, public = %local.listen.public, internal = %local.listen.internal, "serving"); + tracing::info!(target: "weft_runtime", roles = ?hosted, public = %local.listen.public, internal = %local.listen.internal, containers = ?containers, "serving"); // The internal routes are reached on the public port too, for // a caller outside the machine (`Vantage::Public`). A local // install trusts every caller of its public API, and its // internal port is one containers reach, so that port stays - // roles only. + // roles only, and the files door, whose links carry their own + // signature. let front = public.nest(weft_platform_traits::roles::INTERNAL_DOOR, internal.clone()); - let mut servers = vec![ + let mut servers = Vec::new(); + if let Some(listener) = containers_listener { + servers.push(tokio::spawn(server::serve(listener, internal.clone(), aside.clone()))); + } + servers.extend([ tokio::spawn(server::serve(internal_listener, internal, aside.clone())), tokio::spawn(server::serve(public_listener, front, aside.clone())), - ]; + ]); if let (Some(addr), Some(app)) = (local.listen.outside, outside) { let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("bind the outside port {addr}"))?; servers.push(tokio::spawn(server::serve(listener, app, aside.clone()))); diff --git a/crates/weft-runtime/src/log_file.rs b/crates/weft-runtime/src/log_file.rs index 872823d63..8ec454264 100644 --- a/crates/weft-runtime/src/log_file.rs +++ b/crates/weft-runtime/src/log_file.rs @@ -2,9 +2,18 @@ //! [`MAX_BYTES`] it moves aside to `.1` (replacing the one there) //! and a fresh file starts, so a runtime up for months holds at most two //! files' worth of log. +//! +//! The service that starts the runtime points its stdout and stderr at the +//! same file (`weft daemon start`'s command line), for what never passes +//! through the logger: a panic's last words, a library's print. When it +//! does, each roll moves them onto the fresh file too: left on the file +//! moved aside, they would land in `.1`, and after the next roll in +//! a file nobody can open any more. use std::fs::{File, OpenOptions}; use std::io::Write; +use std::os::fd::{AsFd, AsRawFd}; +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::sync::Mutex; @@ -14,6 +23,9 @@ pub const MAX_BYTES: u64 = 50 * 1024 * 1024; pub struct LogFile { path: PathBuf, max_bytes: u64, + /// The process's standard streams that are this file, which each roll + /// moves on to the fresh one. + carried: Vec, state: Mutex, } @@ -30,14 +42,34 @@ impl LogFile { pub fn open(path: &Path, max_bytes: u64) -> std::io::Result { let file = OpenOptions::new().create(true).append(true).open(path)?; let size = file.metadata()?.len(); - Ok(Self { path: path.to_path_buf(), max_bytes, state: Mutex::new(State { file, size, roll_failing: false }) }) + let mut carried = Vec::new(); + for (fd, stream) in [(libc::STDOUT_FILENO, std::io::stdout().as_fd()), (libc::STDERR_FILENO, std::io::stderr().as_fd())] { + if same_file(&file, &File::from(stream.try_clone_to_owned()?))? { + carried.push(fd); + } + } + Ok(Self { path: path.to_path_buf(), max_bytes, carried, state: Mutex::new(State { file, size, roll_failing: false }) }) } fn write_all(&self, buf: &[u8]) -> std::io::Result<()> { let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); if state.size > 0 && state.size + buf.len() as u64 > self.max_bytes { match self.roll() { - Ok(fresh) => { + Ok(mut fresh) => { + for &fd in &self.carried { + // SAFETY: both descriptors are open in this process; + // `dup2` only repoints `fd` at the fresh file. + if unsafe { libc::dup2(fresh.as_raw_fd(), fd) } < 0 { + // Said in the log itself: the stream is on the + // file just moved aside, which nobody reads. + let e = std::io::Error::last_os_error(); + let _ = writeln!( + fresh, + "weft-runtime: could not move descriptor {fd} onto the fresh log ({e}); what it prints from now on lands in {}.1", + self.path.display() + ); + } + } state.file = fresh; state.size = 0; state.roll_failing = false; @@ -67,6 +99,12 @@ impl LogFile { } } +/// Whether `a` and `b` are open on the same file. +fn same_file(a: &File, b: &File) -> std::io::Result { + let (a, b) = (a.metadata()?, b.metadata()?); + Ok(a.dev() == b.dev() && a.ino() == b.ino()) +} + /// One `tracing` write, handed to the shared file. pub struct LogWriter<'a>(&'a LogFile); @@ -91,6 +129,19 @@ impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogFile { mod tests { use super::*; + /// A file is told apart from another one, and from itself opened twice + /// (how the log tells whether stderr is it). + #[test] + fn the_same_file_opened_twice_is_one_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("runtime.log"); + let one = OpenOptions::new().create(true).append(true).open(&path).unwrap(); + let again = OpenOptions::new().append(true).open(&path).unwrap(); + let other = File::create(dir.path().join("other")).unwrap(); + assert!(same_file(&one, &again).unwrap()); + assert!(!same_file(&one, &other).unwrap()); + } + #[test] fn a_full_log_moves_aside_and_starts_again() { let dir = tempfile::tempdir().unwrap(); diff --git a/deploy/terraform/gcp/outputs.tf b/deploy/terraform/gcp/outputs.tf index 1af98ce1b..98535be00 100644 --- a/deploy/terraform/gcp/outputs.tf +++ b/deploy/terraform/gcp/outputs.tf @@ -6,6 +6,11 @@ output "registry" { value = local.images_registry } +output "files_bucket" { + description = "The bucket the install keeps its files in, where the install workflow stores the weft CLI it ran with (under install/cli//) for a project's deploy to download." + value = google_storage_bucket.files.name +} + output "url" { description = "The install's address: the dispatcher's own Cloud Run address, which answers with no domain at all." value = local.role_urls["dispatcher"] diff --git a/deploy/terraform/gcp/serverless.tf b/deploy/terraform/gcp/serverless.tf index f655ef8bf..674a5e76c 100644 --- a/deploy/terraform/gcp/serverless.tf +++ b/deploy/terraform/gcp/serverless.tf @@ -28,6 +28,23 @@ locals { dispatcher = var.dispatcher_max_copies broker = var.broker_max_copies } + # The memory of one copy of each role, in Mi, read from the setting + # (`Mi` or `Gi`); 0 when it is neither, which the role's + # precondition refuses. + role_memory = { + dispatcher = var.dispatcher_memory + broker = var.broker_memory + listener = var.listener_memory + supervisor = var.supervisor_memory + } + # SYNC: reading `Mi` / `Gi` <-> variables.tf (holder_memory's validation) + role_memory_mi = { + for role, memory in local.role_memory : role => try( + tonumber(regex("^([0-9]+)Mi$", memory)[0]), + tonumber(regex("^([0-9]+)Gi$", memory)[0]) * 1024, + 0, + ) + } # How many requests one copy of a role takes at once, where it is not # Cloud Run's default of 80. Every worker, listener and holder process # keeps one line open to the broker (a websocket, which Cloud Run counts @@ -252,6 +269,9 @@ resource "google_cloud_run_v2_service" "role" { # that wrote has answered, which needs its CPU then. Billed while # an instance is up, and none is up while nothing calls it. cpu_idle = !contains(local.writers, each.value) + limits = { + memory = local.role_memory[each.value] + } } env { @@ -292,6 +312,16 @@ resource "google_cloud_run_v2_service" "role" { } } + lifecycle { + # A role runs on Cloud Run's default of one CPU, which takes up to + # 4Gi; it is never given less than Cloud Run's default of 512Mi. + # SYNC: the memory rule <-> variables.tf (dispatcher_memory, broker_memory, listener_memory, supervisor_memory) + precondition { + condition = local.role_memory_mi[each.value] >= 512 && local.role_memory_mi[each.value] <= 4096 + error_message = "${each.value}_memory is \"${local.role_memory[each.value]}\": it is a whole number of Mi or Gi from 512Mi to 4Gi (\"512Mi\", \"1Gi\", \"1536Mi\"). 512Mi is Cloud Run's default and what a role is never given less than, and above 4Gi Cloud Run needs more than the one CPU a role runs on." + } + } + # A role calls Google as the core account the moment it is up (the # broker reaches the object store as it boots, and the first pass of the # broker and the dispatcher sets wakes and, on the dispatcher, sizes the diff --git a/deploy/terraform/gcp/variables.tf b/deploy/terraform/gcp/variables.tf index 4c8125abb..efa068102 100644 --- a/deploy/terraform/gcp/variables.tf +++ b/deploy/terraform/gcp/variables.tf @@ -1,4 +1,4 @@ -# SYNC: the variables the install workflow sets (project_id, region, zone, database_url, database_listen_url, cloud_sql, cloud_sql_tier, cloud_sql_max_connections, dispatcher_max_copies, broker_max_copies, runtime_image, builder_base_image, weft_repository, weft_commit, access_apps_json, alert_email) <-> .github/workflows/install-gcp.yml (the TF_VAR_* names) +# SYNC: the variables the install workflow sets (project_id, region, zone, database_url, database_listen_url, cloud_sql, cloud_sql_tier, cloud_sql_max_connections, dispatcher_max_copies, broker_max_copies, dispatcher_memory, broker_memory, listener_memory, supervisor_memory, holder_memory, runtime_image, builder_base_image, weft_repository, weft_commit, access_apps_json, alert_email) <-> .github/workflows/install-gcp.yml (the TF_VAR_* names) variable "project_id" { description = "The GCP project the install lives in." @@ -80,6 +80,34 @@ variable "broker_max_copies" { } } +# The memory of one copy of each role. Cloud Run's own default is 512Mi, +# and a role left at it is given 512Mi again by every apply, so each one +# is set here, where a value somebody chose survives the next install run. +# SYNC: the memory rule <-> serverless.tf (google_cloud_run_v2_service.role's precondition) +variable "dispatcher_memory" { + description = "The memory of each copy of the dispatcher, in Mi or Gi (\"1Gi\", \"1536Mi\"), from 512Mi to 4Gi. More than the other roles by default: every version is compiled there, and on Cloud Run the files a build writes are held in memory, beside the programs the dispatcher keeps read." + type = string + default = "1Gi" +} + +variable "broker_memory" { + description = "The memory of each copy of the broker, in Mi or Gi, from 512Mi to 4Gi." + type = string + default = "512Mi" +} + +variable "listener_memory" { + description = "The memory of each copy of the listener, in Mi or Gi, from 512Mi to 4Gi." + type = string + default = "512Mi" +} + +variable "supervisor_memory" { + description = "The memory of each copy of the supervisor, in Mi or Gi, from 512Mi to 4Gi." + type = string + default = "512Mi" +} + variable "database_listen_url" { description = "A direct (session) address of the same database, when database_url goes through a pooler that hands out a connection per transaction: a LISTEN needs a session of its own (the runtime refuses to start, naming this, when it cannot listen). Empty when database_url is already one." type = string @@ -122,9 +150,20 @@ variable "holder_cpu" { } variable "holder_memory" { - description = "The memory of each holder." + description = "The memory of each holder, in Mi or Gi (\"512Mi\", \"1Gi\"), at least 512Mi; Cloud Run refuses more than its CPU (`holder_cpu`) takes." type = string default = "512Mi" + # The roles' 512Mi floor, without their 4Gi ceiling: a holder's CPU is + # its own setting, and Cloud Run refuses more memory than that CPU takes. + # SYNC: reading `Mi` / `Gi` <-> serverless.tf (local.role_memory_mi) + validation { + condition = try( + tonumber(regex("^([0-9]+)Mi$", var.holder_memory)[0]), + tonumber(regex("^([0-9]+)Gi$", var.holder_memory)[0]) * 1024, + 0, + ) >= 512 + error_message = "holder_memory is a whole number of Mi or Gi of at least 512Mi (\"512Mi\", \"1Gi\"), Cloud Run's default and what a holder is never given less than." + } } variable "runtime_image" { diff --git a/docs/src/nodes/storage.md b/docs/src/nodes/storage.md index 8daaa9936..130a3363f 100644 --- a/docs/src/nodes/storage.md +++ b/docs/src/nodes/storage.md @@ -172,10 +172,11 @@ who will open it: | Call | Use it when | |---|---| | `internal_link(&file)` | Code inside the install opens it: a script or a process your node runs on the worker. The link goes straight to storage, never out through a tunnel and back | +| `container_link(&file)` | One of your project's own containers opens it: an infra unit your node hands the file to. Inside a container `127.0.0.1` is the container itself, so on a local install this link reaches your machine at `host.docker.internal`, through weft's internal port. On a cloud install it is the same link as `internal_link` | | `external_link(&file)` | Anything outside the install opens it: a browser, an app, a provider's API. This is the link a `Reply` puts in its answer in place of a file | | `external_file(&file)` | You are sending the file to an outside service: its external link when the open internet can open it, else a `data:` URL carrying the bytes, with the file's mime type and filename beside it | -Both links come back as a `FileLink { url, public }`, where `public` says +Every link comes back as a `FileLink { url, public }`, where `public` says whether the open internet can open `url`. An external link goes the first of these ways that exists: diff --git a/docs/src/running/cloud.md b/docs/src/running/cloud.md index 38e39aa2d..427be1c43 100644 --- a/docs/src/running/cloud.md +++ b/docs/src/running/cloud.md @@ -187,6 +187,16 @@ share of at least 6 and a dispatcher copy 5, and one whose share is smaller refuses to start, with a log line that names the numbers and these variables. +If you want a role to have more memory, set `WEFT_DISPATCHER_MEMORY` (1Gi +when unset), `WEFT_BROKER_MEMORY`, `WEFT_LISTENER_MEMORY` or +`WEFT_SUPERVISOR_MEMORY` (512Mi each when unset), a whole number of `Mi` or +`Gi` from 512Mi to 4Gi, or `WEFT_HOLDER_MEMORY` for each copy that holds a +trigger's open connection (512Mi when unset, at least 512Mi). The install sets each one on every run, so a value you +change by hand in the Cloud Run console is put back the next time the workflow +runs. The dispatcher gets more because every version is compiled there, and on +Cloud Run the files a build writes are held in memory, beside the programs the +dispatcher keeps read (about 90 MiB at most). + These are repository variables on your fork, like the ones above, so merging a new weft into the fork keeps them. After you change one, run the install workflow again. @@ -212,7 +222,11 @@ run it right after weft's main moved and your fork caught up, the release for that source may still be building: the workflow waits for it rather than compiling the same thing, and says so in its log. When it does have to build, it keeps the compiled dependencies for the next run, so after an -upgrade only what changed in weft compiles again. +upgrade only what changed in weft compiles again. Among the images is the +standard worker, which every project made only of the standard nodes runs on, +so the first of them to run after an upgrade starts at once instead of +waiting a few minutes for that build. It also keeps the CLI it used in your +install, so your projects' deploys download it instead of compiling it. When it finishes, its last step (in the run's summary, and in its log for `gh run view --log`) gives you the install's address, a Cloud Run @@ -683,9 +697,12 @@ frontend's token, with the install's address, to a file only you can read. Put those in the frontend's environment. `weft ci add` writes `.github/workflows/deploy.yml`, a workflow you run by -hand from the Actions tab. It takes the weft CLI the release built from the -source your install runs (waiting for it while that release is still -building, or building it when the release has none), deploys +hand from the Actions tab. It takes the weft CLI built from the source your +install runs: first the one the install workflow stored in your install +(signed in with `WEFT_OPERATOR_KEY`, its sha256 checked), which is how a fork +gets its own CLI without compiling it, then the one weft's release built +(waiting for it while that release is still building), and it compiles the +CLI only when neither has one. Then it deploys the program with `weft infra upgrade --all` then `weft resync --all`, then builds `front/` with Docker and deploys it to the repository's service. If you run `weft ci add` again, it replaces the file only if you have not edited diff --git a/packages/weft-graph/src/protocol.ts b/packages/weft-graph/src/protocol.ts index f8fbd60ed..86306b965 100644 --- a/packages/weft-graph/src/protocol.ts +++ b/packages/weft-graph/src/protocol.ts @@ -2077,6 +2077,16 @@ export interface LiveDataItem { payload?: unknown; confirm?: string; }; + /// On a secret the node hands out once: the action kind pressed as soon + /// as the value is copied out, with `{ value }` as its payload, so the + /// node retires the value its reader now holds. + // SYNC: taken <-> crates/weft-core/src/live.rs LiveItem::taken, LiveItem::taken_press + taken?: string; + /// The line standing where such a secret was once it was handed over; + /// its button makes a new one. + // SYNC: handedOver <-> crates/weft-core/src/live.rs LiveItem::handed_over, + // catalog/postgres/database/images/credential/bootstrap.py (password_item) + handedOver?: boolean; } /// State of one node's display. The dispatcher pushes one of these diff --git a/packages/weft-graph/src/webview/lib/components/project/ProjectNode.svelte b/packages/weft-graph/src/webview/lib/components/project/ProjectNode.svelte index 46f31a2bd..347ab5d06 100644 --- a/packages/weft-graph/src/webview/lib/components/project/ProjectNode.svelte +++ b/packages/weft-graph/src/webview/lib/components/project/ProjectNode.svelte @@ -413,6 +413,20 @@ /// closing/reopening the inspector resets to hidden, which is /// the desired security default. let revealedSecrets = $state>({}); + + /// Tell the node a secret it hands out once was copied out (`taken`), + /// with the exact value copied as the proof it asks for, through the + /// same press a button sends. No confirm: the copy already happened. + // SYNC: takenFrom <-> crates/weft-core/src/live.rs LiveItem::taken_press, + // catalog/postgres/database/images/credential/bootstrap.py (TAKEN_ACTION, taken) + function takenFrom(item: LiveDataItem, copied: string) { + if (!item.taken) return; + window.dispatchEvent( + new CustomEvent('weft-display-action', { + detail: { nodeId: id, actionKind: item.taken, payload: { value: copied } }, + }), + ); + } // The node's root element (`bind:this`). `$state` because an `$effect` below // reads it (the deselect-blur effect): when `bind:this` assigns it after // mount, the effect must re-run, which a plain `let` would not trigger. @@ -1586,6 +1600,8 @@ preview) rendered the SAME way in the full body and the simplified card, so the two never drift. `actionBtn` is its helper. Defined at the top level so both render branches can call it. --> + {#snippet actionBtn(item: LiveDataItem)} {#if item.action} - + takenFrom(item, copied) : undefined} + /> {@render actionBtn(item)} diff --git a/packages/weft-graph/src/webview/lib/components/ui/CopyButton.svelte b/packages/weft-graph/src/webview/lib/components/ui/CopyButton.svelte index 7679fd2e5..34dc3ade6 100644 --- a/packages/weft-graph/src/webview/lib/components/ui/CopyButton.svelte +++ b/packages/weft-graph/src/webview/lib/components/ui/CopyButton.svelte @@ -2,41 +2,69 @@ // `label` names WHAT is copied (e.g. "Copy run state"): it becomes the // resting tooltip and the copied-confirmation. Omitted, the button falls back // to the generic "Copy" / "Copied!". + // `oncopied` hears each copy once the text is on the clipboard, with the + // exact text that got there (the one read when the press started). let { text, label, - class: className = '' - }: { text: string; label?: string; class?: string } = $props(); - let copied = $state(false); + class: className = '', + oncopied + }: { text: string; label?: string; class?: string; oncopied?: (copied: string) => void } = $props(); + // What the last press did, shown in the tooltip for a moment. + let outcome = $state<'copied' | 'failed' | null>(null); const restingTitle = $derived(label ?? 'Copy'); const copiedTitle = $derived(label ? `${label} (copied!)` : 'Copied!'); + const failedTitle = $derived(label ? `${label} (could not copy)` : 'Could not copy'); - async function copy() { + /// Put `value` on the clipboard: whether it got there. + async function written(value: string): Promise { try { - await navigator.clipboard.writeText(text); + await navigator.clipboard.writeText(value); + return true; } catch { - // Fallback for Chrome iframe/embed restrictions + // Fallback for Chrome iframe/embed restrictions, which says + // whether the copy happened by what it returns. const ta = document.createElement('textarea'); - ta.value = text; + ta.value = value; ta.style.position = 'fixed'; ta.style.opacity = '0'; document.body.appendChild(ta); - ta.select(); - document.execCommand('copy'); - document.body.removeChild(ta); + try { + ta.select(); + return document.execCommand('copy'); + } catch { + return false; + } finally { + document.body.removeChild(ta); + } } - copied = true; - setTimeout(() => copied = false, 1500); + } + + async function copy() { + // Read once: the prop may change while the clipboard is written. + const value = text; + const done = await written(value); + outcome = done ? 'copied' : 'failed'; + // Heard only when the text is really on the clipboard, and with + // what got there: a secret its node hands out once is retired on + // this, against the value its reader now holds. + if (done) oncopied?.(value); + setTimeout(() => outcome = null, 1500); }