From 92e5a4bfd8fc37280ba2782df8208577768e9696 Mon Sep 17 00:00:00 2001 From: Grivn Date: Mon, 26 Oct 2026 22:07:21 +0800 Subject: [PATCH 1/7] feat(memory): import the v0.3.0 AML backend Add a separate revisioned raw-record store and an explicit memory-agent backend: bounded BM25 retrieval, System 2 query planning, JEV evidence judgments, and a time-ordered View. Background layered consolidation links its topics, values and instructions back to the original records. Include the per-user Add/Search adapter, supervised workers, provider budgets and retries, authenticated inspection, metrics, deployment tooling, and the deterministic tests of the evaluated baseline. Records retain opaque payloads and revision/CAS semantics; JEV request limits use UTF-16 units, and historical evidence keeps its correction/time boundaries. Keep the current public CLI search helpers independent from the frozen retrieval primitives from this first milestone. Adjust the architecture oracle and remove a deployment-test dependency on excluded research docs. Internal research/prototype files and reverted experiments are not imported. Evaluation-Commit: b2d0b0e05e5185f45b77d8e04f128aecf4bfd4ee Archived-Milestone: 13dda8ab0da574358dca90cbf273d68487861470 --- Makefile | 18 +- README.md | 25 + SECURITY.md | 32 +- cmd/memory/aml.go | 307 ++ cmd/memory/aml_admin_bind_test.go | 93 + cmd/memory/aml_holds_test.go | 308 ++ cmd/memory/aml_inspect.go | 264 ++ cmd/memory/aml_inspect_test.go | 241 ++ cmd/memory/aml_process.go | 549 ++++ cmd/memory/aml_test.go | 1935 ++++++++++++ cmd/memory/aml_watch.go | 1264 ++++++++ cmd/memory/aml_watch_test.go | 2751 +++++++++++++++++ cmd/memory/records.go | 100 + cmd/memory/serve.go | 236 ++ cmd/memory/serve_dispatch_test.go | 58 + cmd/root.go | 19 +- cmd/root_test.go | 108 + deploy/aml/Caddyfile.dedicated | 49 + deploy/aml/Caddyfile.shared | 46 + deploy/aml/README.md | 275 ++ deploy/aml/aml-inspect@.service | 71 + deploy/aml/aml-serve@.service | 71 + deploy/aml/aml-watch@.service | 42 + deploy/aml/aml-watch@.timer | 16 + deploy/aml/instance.sh | 1225 ++++++++ deploy/aml/memory-config.industrial.json | 34 + deploy/aml/preflight.sh | 544 ++++ deploy/aml/wg0.conf.example | 25 + docs/admin-api.md | 445 +++ docs/aml-adapter.md | 1347 ++++++++ docs/inspect-api.md | 601 ++++ docs/memory-service.md | 348 +++ docs/records-indexes.md | 155 + go.mod | 18 +- go.sum | 65 +- internal/aml/admin.go | 373 +++ internal/aml/admin_test.go | 778 +++++ internal/aml/background_test.go | 1207 ++++++++ internal/aml/budgets.go | 344 +++ internal/aml/budgets_test.go | 486 +++ internal/aml/byok.go | 285 ++ internal/aml/byok_test.go | 822 +++++ internal/aml/consolidate_test.go | 234 ++ internal/aml/contract.go | 500 +++ internal/aml/disk_bsd.go | 15 + internal/aml/disk_linux.go | 20 + internal/aml/disk_other.go | 15 + internal/aml/disk_windows.go | 19 + internal/aml/inspect.go | 791 +++++ internal/aml/inspect_caps.go | 250 ++ internal/aml/inspect_caps_test.go | 275 ++ internal/aml/inspect_search.go | 327 ++ internal/aml/inspect_search_test.go | 1032 +++++++ internal/aml/inspect_store.go | 558 ++++ internal/aml/inspect_test.go | 1070 +++++++ internal/aml/inspect_trace.go | 396 +++ internal/aml/metrics.go | 392 +++ internal/aml/metrics_test.go | 673 ++++ internal/aml/middleware.go | 196 ++ internal/aml/middleware_test.go | 627 ++++ internal/aml/modelcache_test.go | 47 + internal/aml/order_test.go | 163 + internal/aml/owner_other.go | 9 + internal/aml/owner_unix.go | 18 + internal/aml/private_files.go | 231 ++ internal/aml/registry.go | 237 ++ internal/aml/registry_test.go | 25 + internal/aml/scheduler.go | 646 ++++ internal/aml/scheduler_test.go | 1035 +++++++ internal/aml/search_test.go | 244 ++ internal/aml/server.go | 1479 +++++++++ internal/aml/server_test.go | 1014 ++++++ internal/aml/stores.go | 413 +++ internal/aml/stores_test.go | 741 +++++ internal/localrpc/serve.go | 112 + internal/memory/service/service.go | 130 + internal/memory/service/service_test.go | 69 + internal/memoryagent/adapter_test.go | 207 ++ internal/memoryagent/agent_test.go | 485 +++ internal/memoryagent/backlog.go | 77 + internal/memoryagent/backlog_test.go | 109 + internal/memoryagent/budget.go | 500 +++ internal/memoryagent/budget_caps_test.go | 164 + internal/memoryagent/config.go | 117 + internal/memoryagent/consolidate.go | 322 ++ internal/memoryagent/conversation.go | 271 ++ internal/memoryagent/conversation_test.go | 524 ++++ internal/memoryagent/index_vectors.go | 73 + internal/memoryagent/keys.go | 331 ++ internal/memoryagent/keys_test.go | 444 +++ internal/memoryagent/layered.go | 533 ++++ internal/memoryagent/layered_link.go | 533 ++++ internal/memoryagent/layered_link_test.go | 532 ++++ internal/memoryagent/layered_test.go | 1194 +++++++ internal/memoryagent/modelcache.go | 142 + internal/memoryagent/modelcache_test.go | 768 +++++ internal/memoryagent/models.go | 149 + internal/memoryagent/overlay.go | 266 ++ internal/memoryagent/provider.go | 725 +++++ internal/memoryagent/provider_test.go | 1329 ++++++++ internal/memoryagent/read.go | 641 ++++ internal/memoryagent/reader.go | 234 ++ internal/memoryagent/reader_test.go | 313 ++ internal/memoryagent/reading.go | 373 +++ internal/memoryagent/records.go | 389 +++ internal/memoryagent/resilience.go | 1338 ++++++++ internal/memoryagent/resilience_close_test.go | 253 ++ internal/memoryagent/resilience_test.go | 853 +++++ .../memoryagent/resilience_throttle_test.go | 614 ++++ .../memoryagent/resilience_transport_test.go | 299 ++ .../memoryagent/resilience_unstable_test.go | 644 ++++ internal/memoryagent/simple.go | 199 ++ internal/memoryagent/simple_compose.go | 276 ++ internal/memoryagent/simple_compose_test.go | 522 ++++ internal/memoryagent/simple_degraded_test.go | 127 + internal/memoryagent/simple_judge.go | 303 ++ internal/memoryagent/simple_loop.go | 307 ++ internal/memoryagent/simple_loop_test.go | 789 +++++ internal/memoryagent/simple_order.go | 107 + internal/memoryagent/simple_order_test.go | 232 ++ internal/memoryagent/simple_plan.go | 258 ++ internal/memoryagent/simple_pool.go | 184 ++ internal/memoryagent/simple_questions.go | 52 + internal/memoryagent/simple_test.go | 738 +++++ internal/memoryagent/sqlite_uri_test.go | 78 + internal/memoryagent/timelines.go | 1141 +++++++ internal/memoryagent/timelines_layer.go | 695 +++++ internal/memoryagent/timelines_layer_test.go | 890 ++++++ internal/memoryagent/timelines_store.go | 102 + internal/memoryagent/timelines_test.go | 1076 +++++++ internal/memoryagent/trace.go | 329 ++ internal/memoryagent/trace_test.go | 292 ++ internal/records/LICENSE.v8 | 61 + internal/records/cache.go | 196 ++ internal/records/derived.go | 213 ++ internal/records/derived_test.go | 625 ++++ internal/records/digest.go | 187 ++ internal/records/digest_test.go | 441 +++ internal/records/fts.go | 261 ++ internal/records/fts_test.go | 255 ++ internal/records/golden_test.go | 187 ++ internal/records/index.go | 348 +++ internal/records/index_build.go | 246 ++ internal/records/index_golden_test.go | 181 ++ internal/records/index_relations.go | 92 + internal/records/index_test.go | 229 ++ internal/records/isolation_test.go | 81 + internal/records/jslog.go | 93 + internal/records/lexical.go | 379 +++ internal/records/native_bm25.go | 51 + internal/records/native_bm25_test.go | 92 + internal/records/neighbors.go | 269 ++ internal/records/neighbors_test.go | 390 +++ internal/records/norace_test.go | 6 + internal/records/prepared_lexical.go | 201 ++ internal/records/query.go | 247 ++ internal/records/race_test.go | 6 + internal/records/read.go | 162 + internal/records/reader.go | 372 +++ internal/records/reader_test.go | 592 ++++ internal/records/recall.go | 434 +++ internal/records/recall_test.go | 123 + internal/records/runtime_test.go | 226 ++ internal/records/selection.go | 304 ++ internal/records/sequence.go | 46 + internal/records/sequence_test.go | 41 + internal/records/serve.go | 170 + internal/records/serve_test.go | 247 ++ internal/records/sqlite_uri.go | 54 + internal/records/sqlite_uri_test.go | 288 ++ internal/records/store.go | 372 +++ internal/records/store_test.go | 117 + internal/records/testdata/bm25-rounding.json | 827 +++++ internal/records/testdata/golden.json | 727 +++++ internal/records/testdata/index-routes.golden | 368 +++ internal/records/traversal_test.go | 473 +++ internal/records/vectors.go | 441 +++ internal/records/vectors_test.go | 163 + internal/retrieval/entities.go | 250 ++ internal/retrieval/intent.go | 80 + internal/retrieval/intent_languages.go | 124 + internal/retrieval/tokens.go | 91 + scripts/aml/README.md | 148 + scripts/aml/drill.sh | 761 +++++ scripts/aml/fakeprovider/main.go | 687 ++++ scripts/aml/fakeprovider/replies.go | 426 +++ scripts/aml/final.sh | 145 + scripts/aml/knee.jq | 38 + scripts/aml/lib.sh | 324 ++ scripts/aml/load.sh | 493 +++ scripts/aml/parity/README.md | 158 + scripts/aml/parity/replay.py | 1096 +++++++ scripts/aml/poisson.jq | 25 + scripts/aml/scenario.jq | 130 + test/deploy/aml_scripts_test.go | 1893 ++++++++++++ test/deploy/fakeprovider_test.go | 554 ++++ test/deploy/inspect_script_test.go | 286 ++ test/deploy/instance_script_test.go | 1737 +++++++++++ test/deploy/preflight_script_test.go | 571 ++++ .../architecture/mnemond_structure_test.go | 9 +- .../architecture/release_boundary_test.go | 4 +- .../architecture/repository_hygiene_test.go | 7 + 202 files changed, 78327 insertions(+), 20 deletions(-) create mode 100644 cmd/memory/aml.go create mode 100644 cmd/memory/aml_admin_bind_test.go create mode 100644 cmd/memory/aml_holds_test.go create mode 100644 cmd/memory/aml_inspect.go create mode 100644 cmd/memory/aml_inspect_test.go create mode 100644 cmd/memory/aml_process.go create mode 100644 cmd/memory/aml_test.go create mode 100644 cmd/memory/aml_watch.go create mode 100644 cmd/memory/aml_watch_test.go create mode 100644 cmd/memory/records.go create mode 100644 cmd/memory/serve.go create mode 100644 cmd/memory/serve_dispatch_test.go create mode 100644 deploy/aml/Caddyfile.dedicated create mode 100644 deploy/aml/Caddyfile.shared create mode 100644 deploy/aml/README.md create mode 100644 deploy/aml/aml-inspect@.service create mode 100644 deploy/aml/aml-serve@.service create mode 100644 deploy/aml/aml-watch@.service create mode 100644 deploy/aml/aml-watch@.timer create mode 100755 deploy/aml/instance.sh create mode 100644 deploy/aml/memory-config.industrial.json create mode 100755 deploy/aml/preflight.sh create mode 100644 deploy/aml/wg0.conf.example create mode 100644 docs/admin-api.md create mode 100644 docs/aml-adapter.md create mode 100644 docs/inspect-api.md create mode 100644 docs/memory-service.md create mode 100644 docs/records-indexes.md create mode 100644 internal/aml/admin.go create mode 100644 internal/aml/admin_test.go create mode 100644 internal/aml/background_test.go create mode 100644 internal/aml/budgets.go create mode 100644 internal/aml/budgets_test.go create mode 100644 internal/aml/byok.go create mode 100644 internal/aml/byok_test.go create mode 100644 internal/aml/consolidate_test.go create mode 100644 internal/aml/contract.go create mode 100644 internal/aml/disk_bsd.go create mode 100644 internal/aml/disk_linux.go create mode 100644 internal/aml/disk_other.go create mode 100644 internal/aml/disk_windows.go create mode 100644 internal/aml/inspect.go create mode 100644 internal/aml/inspect_caps.go create mode 100644 internal/aml/inspect_caps_test.go create mode 100644 internal/aml/inspect_search.go create mode 100644 internal/aml/inspect_search_test.go create mode 100644 internal/aml/inspect_store.go create mode 100644 internal/aml/inspect_test.go create mode 100644 internal/aml/inspect_trace.go create mode 100644 internal/aml/metrics.go create mode 100644 internal/aml/metrics_test.go create mode 100644 internal/aml/middleware.go create mode 100644 internal/aml/middleware_test.go create mode 100644 internal/aml/modelcache_test.go create mode 100644 internal/aml/order_test.go create mode 100644 internal/aml/owner_other.go create mode 100644 internal/aml/owner_unix.go create mode 100644 internal/aml/private_files.go create mode 100644 internal/aml/registry.go create mode 100644 internal/aml/registry_test.go create mode 100644 internal/aml/scheduler.go create mode 100644 internal/aml/scheduler_test.go create mode 100644 internal/aml/search_test.go create mode 100644 internal/aml/server.go create mode 100644 internal/aml/server_test.go create mode 100644 internal/aml/stores.go create mode 100644 internal/aml/stores_test.go create mode 100644 internal/localrpc/serve.go create mode 100644 internal/memory/service/service.go create mode 100644 internal/memory/service/service_test.go create mode 100644 internal/memoryagent/adapter_test.go create mode 100644 internal/memoryagent/agent_test.go create mode 100644 internal/memoryagent/backlog.go create mode 100644 internal/memoryagent/backlog_test.go create mode 100644 internal/memoryagent/budget.go create mode 100644 internal/memoryagent/budget_caps_test.go create mode 100644 internal/memoryagent/config.go create mode 100644 internal/memoryagent/consolidate.go create mode 100644 internal/memoryagent/conversation.go create mode 100644 internal/memoryagent/conversation_test.go create mode 100644 internal/memoryagent/index_vectors.go create mode 100644 internal/memoryagent/keys.go create mode 100644 internal/memoryagent/keys_test.go create mode 100644 internal/memoryagent/layered.go create mode 100644 internal/memoryagent/layered_link.go create mode 100644 internal/memoryagent/layered_link_test.go create mode 100644 internal/memoryagent/layered_test.go create mode 100644 internal/memoryagent/modelcache.go create mode 100644 internal/memoryagent/modelcache_test.go create mode 100644 internal/memoryagent/models.go create mode 100644 internal/memoryagent/overlay.go create mode 100644 internal/memoryagent/provider.go create mode 100644 internal/memoryagent/provider_test.go create mode 100644 internal/memoryagent/read.go create mode 100644 internal/memoryagent/reader.go create mode 100644 internal/memoryagent/reader_test.go create mode 100644 internal/memoryagent/reading.go create mode 100644 internal/memoryagent/records.go create mode 100644 internal/memoryagent/resilience.go create mode 100644 internal/memoryagent/resilience_close_test.go create mode 100644 internal/memoryagent/resilience_test.go create mode 100644 internal/memoryagent/resilience_throttle_test.go create mode 100644 internal/memoryagent/resilience_transport_test.go create mode 100644 internal/memoryagent/resilience_unstable_test.go create mode 100644 internal/memoryagent/simple.go create mode 100644 internal/memoryagent/simple_compose.go create mode 100644 internal/memoryagent/simple_compose_test.go create mode 100644 internal/memoryagent/simple_degraded_test.go create mode 100644 internal/memoryagent/simple_judge.go create mode 100644 internal/memoryagent/simple_loop.go create mode 100644 internal/memoryagent/simple_loop_test.go create mode 100644 internal/memoryagent/simple_order.go create mode 100644 internal/memoryagent/simple_order_test.go create mode 100644 internal/memoryagent/simple_plan.go create mode 100644 internal/memoryagent/simple_pool.go create mode 100644 internal/memoryagent/simple_questions.go create mode 100644 internal/memoryagent/simple_test.go create mode 100644 internal/memoryagent/sqlite_uri_test.go create mode 100644 internal/memoryagent/timelines.go create mode 100644 internal/memoryagent/timelines_layer.go create mode 100644 internal/memoryagent/timelines_layer_test.go create mode 100644 internal/memoryagent/timelines_store.go create mode 100644 internal/memoryagent/timelines_test.go create mode 100644 internal/memoryagent/trace.go create mode 100644 internal/memoryagent/trace_test.go create mode 100644 internal/records/LICENSE.v8 create mode 100644 internal/records/cache.go create mode 100644 internal/records/derived.go create mode 100644 internal/records/derived_test.go create mode 100644 internal/records/digest.go create mode 100644 internal/records/digest_test.go create mode 100644 internal/records/fts.go create mode 100644 internal/records/fts_test.go create mode 100644 internal/records/golden_test.go create mode 100644 internal/records/index.go create mode 100644 internal/records/index_build.go create mode 100644 internal/records/index_golden_test.go create mode 100644 internal/records/index_relations.go create mode 100644 internal/records/index_test.go create mode 100644 internal/records/isolation_test.go create mode 100644 internal/records/jslog.go create mode 100644 internal/records/lexical.go create mode 100644 internal/records/native_bm25.go create mode 100644 internal/records/native_bm25_test.go create mode 100644 internal/records/neighbors.go create mode 100644 internal/records/neighbors_test.go create mode 100644 internal/records/norace_test.go create mode 100644 internal/records/prepared_lexical.go create mode 100644 internal/records/query.go create mode 100644 internal/records/race_test.go create mode 100644 internal/records/read.go create mode 100644 internal/records/reader.go create mode 100644 internal/records/reader_test.go create mode 100644 internal/records/recall.go create mode 100644 internal/records/recall_test.go create mode 100644 internal/records/runtime_test.go create mode 100644 internal/records/selection.go create mode 100644 internal/records/sequence.go create mode 100644 internal/records/sequence_test.go create mode 100644 internal/records/serve.go create mode 100644 internal/records/serve_test.go create mode 100644 internal/records/sqlite_uri.go create mode 100644 internal/records/sqlite_uri_test.go create mode 100644 internal/records/store.go create mode 100644 internal/records/store_test.go create mode 100644 internal/records/testdata/bm25-rounding.json create mode 100644 internal/records/testdata/golden.json create mode 100644 internal/records/testdata/index-routes.golden create mode 100644 internal/records/traversal_test.go create mode 100644 internal/records/vectors.go create mode 100644 internal/records/vectors_test.go create mode 100644 internal/retrieval/entities.go create mode 100644 internal/retrieval/intent.go create mode 100644 internal/retrieval/intent_languages.go create mode 100644 internal/retrieval/tokens.go create mode 100644 scripts/aml/README.md create mode 100755 scripts/aml/drill.sh create mode 100644 scripts/aml/fakeprovider/main.go create mode 100644 scripts/aml/fakeprovider/replies.go create mode 100755 scripts/aml/final.sh create mode 100644 scripts/aml/knee.jq create mode 100644 scripts/aml/lib.sh create mode 100755 scripts/aml/load.sh create mode 100644 scripts/aml/parity/README.md create mode 100755 scripts/aml/parity/replay.py create mode 100644 scripts/aml/poisson.jq create mode 100644 scripts/aml/scenario.jq create mode 100644 test/deploy/aml_scripts_test.go create mode 100644 test/deploy/fakeprovider_test.go create mode 100644 test/deploy/inspect_script_test.go create mode 100644 test/deploy/instance_script_test.go create mode 100644 test/deploy/preflight_script_test.go diff --git a/Makefile b/Makefile index 3274c5c6f..dd20784d2 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,12 @@ DETERMINISTIC_PKGS := \ ./internal/agency/attach \ ./internal/agency/authority \ ./internal/agency/artifact \ + ./internal/records \ + ./internal/retrieval \ + ./internal/localrpc \ + ./internal/memoryagent \ + ./internal/aml \ + ./internal/memory/service \ ./internal/memory/embed \ ./internal/memory/graph \ ./internal/memory/importdraft \ @@ -36,7 +42,8 @@ DETERMINISTIC_PKGS := \ ./web \ ./test/mnemond/architecture \ ./test/mnemond/observer \ - ./test/mnemond/domainops/trace + ./test/mnemond/domainops/trace \ + ./test/deploy/... TESTDATA_PKGS := \ ./testdata/mnemond/domainops/cmd/domain-load \ @@ -44,7 +51,7 @@ TESTDATA_PKGS := \ ./testdata/mnemond/domainops/cmd/domainctl \ ./testdata/mnemond/domainops/world -.PHONY: deps build install uninstall test test-integration test-live +.PHONY: deps build build-linux-amd64 install uninstall test test-integration test-live .PHONY: docker-build docker-run compose-up compose-down compose-dev release-snapshot clean help .DEFAULT_GOAL := help @@ -56,6 +63,10 @@ build: ## Build the mnemon product binary @mkdir -p $(BIN_DIR) go build -ldflags "$(MNEMON_LDFLAGS)" -o $(MNEMON) . +build-linux-amd64: ## Cross-compile a static linux/amd64 binary (AML deployment kit) + @mkdir -p $(BIN_DIR) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X github.com/mnemon-dev/mnemon/cmd.version=$(VERSION)" -o bin/mnemon-linux-amd64 . + install: build ## Install mnemon to GOBIN @mkdir -p $(GOBIN) cp $(MNEMON) $(GOBIN)/mnemon @@ -104,8 +115,9 @@ release-snapshot: ## Build local GoReleaser snapshot artifacts clean: ## Remove build artifacts and test data rm -f mnemon rm -f $(BIN_DIR)/mnemon + rm -f $(BIN_DIR)/mnemon-linux-amd64 rm -rf .testdata help: ## Show this help - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index d82b7719b..f84d12aab 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ Browse the knowledge accumulated by your agents in a standalone local website: ```sh mnemon web # Open http://127.0.0.1:8484 + +Optional: [shared Memory service and paper strategy](docs/memory-service.md), with [composable records indexes](docs/records-indexes.md). ``` Explore a searchable memory library, readable memory pages, a relationship graph, @@ -562,6 +564,27 @@ make help # show all targets See [Development and Deployment](docs/DEPLOYMENT.md) for Docker, Compose, Ollama embedding, and release setup. +## Run as a cloud instance + +Mnemon is also the core runtime engine of a small memory cloud: `mnemon memory-agent +aml-serve` serves the Agent Memory Leaderboard's Add/Search API over one records store per +user, with the memory-agent strategy, an admin API (metrics, readiness, status and a store +overview) and a per-host deploy kit. The website and the console that observes the instances +live in `mnemon-fe`. + +One account is one engine instance, with its own process, API key, admin token, data root, +model budget and limits. On a host with the kit installed, one command opens one: + +```bash +mnemon-instance new beta-alice --layout shared --public-url https://beta.example.com/i/beta-alice --admin-ip 10.77.0.3 +``` + +It prints the base URL and the API key for the account holder, and the admin token and a +registration JSON for the console, once. `mnemon-instance` also suspends, resumes, rotates the +key of and deletes an instance, and deletes one user's data; `mnemon-preflight` checks an +instance before a job. See the [AML adapter runbook](docs/aml-adapter.md) and the +[Admin API](docs/admin-api.md). + ## Documentation - [Agency Preview](docs/AGENCY.md) — maturity boundary, Pi setup, operating model, completion semantics, and optional peers @@ -571,6 +594,8 @@ See [Development and Deployment](docs/DEPLOYMENT.md) for Docker, Compose, Ollama - [Memory Import Guide](docs/IMPORT.md) — schema and LLM prompt for importing historical chats - [Architecture Diagrams](docs/diagrams/) — system architecture, pipelines, lifecycle management - [Memory Agent (experimental)](experimental/memory-agent/) — research preview of Mnemon as a memory agent on DSH, with Jev as its System 1 decision model; paper, code and run records +- [AML adapter runbook](docs/aml-adapter.md) — the Agent Memory Leaderboard service as memory cloud instances: install, open, operate, alerts, data deletion +- [Admin API](docs/admin-api.md) — an instance's admin listener: token, private bind, metrics, readiness, status and store overview ## Star History diff --git a/SECURITY.md b/SECURITY.md index 0c572cc6d..49dcb9f56 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,12 +12,42 @@ We will acknowledge receipt within 48 hours and aim to release a fix within 7 da ## Scope -Mnemon runs locally and stores data in `~/.mnemon/`. Key security considerations: +The Memory CLI runs locally and stores data in `~/.mnemon/`. The AML adapter is a network +service; its model is under "Network service" below. Key security considerations for local use: - **SQLite database** — contains all stored insights; protected by filesystem permissions (`0644`). - **Hook scripts** — shell scripts executed by the LLM CLI at lifecycle events; written with `0755` permissions. - **Embedding provider connection** — optional requests send insight or query text to the configured Ollama or OpenAI-compatible server. The default local Ollama endpoint does not use TLS. If `MNEMON_EMBED_ENDPOINT` points outside a trusted local network, use HTTPS to protect content and any `MNEMON_EMBED_API_KEY` bearer token in transit. +### Network service (memory cloud instances) + +`mnemon memory-agent aml-serve` is a network service: one engine instance per account of the +memory cloud ([runbook](docs/aml-adapter.md)). Its security model: + +- **Data plane.** `POST /add` and `POST /search` require the instance's API key + (`Authorization: Bearer`, `Token`, or `X-Api-Key`), compared in constant time. The data + listener binds loopback; Caddy terminates TLS in front of it. `GET /health` needs no key and + says nothing but `{"status":"ok"}`. Each user has a separate store; one user cannot reach + another's records. +- **Admin plane.** Metrics, readiness, status, the store overview and the profiler are served + on a separate admin listener, never on the data listener. It binds a loopback address, or a + private one only together with an admin token, which every admin request must carry as a + bearer token ([Admin API](docs/admin-api.md)); any other address is refused at startup. +- **Private network.** The instances' admin listeners bind their hosts' WireGuard overlay + addresses. The console on the ops host reaches them over that overlay only, and the instance + hosts' firewalls admit the admin ports only from the hub, on the overlay interface. No admin + port listens on a public address. +- **Secrets.** API keys, admin tokens, provider keys and alert webhooks live in root-only + (0600) environment files on the instance host; the console keeps the admin tokens in its own + 0600 registry and only a fingerprint of each API key. An API key, an admin token or a provider + key never appears in a URL, a log line, a metric label or an error. An alert webhook is a URL + by design (a Telegram bot's token, or a Feishu or WeCom hook's key, is in its path or its + query), so it is never printed, logged or passed as an argument. The host scripts pass the + other secrets to programs only on standard input, never as arguments. +- **Data handling.** Request bodies and user ids are never logged; a user appears only as the + first 8 hex digits of its store's hash. An instance's data is removed with + `mnemon-instance delete`, one user's with `mnemon-instance delete-user`. + ## Supported Versions | Version | Supported | diff --git a/cmd/memory/aml.go b/cmd/memory/aml.go new file mode 100644 index 000000000..0b80f1d44 --- /dev/null +++ b/cmd/memory/aml.go @@ -0,0 +1,307 @@ +package memory + +import ( + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/netip" + "os" + "os/signal" + "path/filepath" + "strconv" + "syscall" + "time" + + "github.com/coreos/go-systemd/v22/daemon" + "github.com/mnemon-dev/mnemon/internal/aml" + "github.com/spf13/cobra" +) + +// newAMLServeCommand is `mnemon memory-agent aml-serve`: the Agent Memory +// Leaderboard Add/Search HTTP contract over one records store per user. +func newAMLServeCommand() *cobra.Command { + command := &cobra.Command{ + Use: "aml-serve", + Short: "Serve the Agent Memory Leaderboard Add/Search HTTP API over per-user stores", + Args: cobra.NoArgs, + RunE: runAMLServe, + } + flags := command.Flags() + flags.String("listen", "127.0.0.1:8090", "TCP address to listen on") + flags.String("data-root", "", "Absolute directory holding one records store per user (required); one aml-serve at a time locks it") + flags.String("memory-config", "", "Absolute path of the memory-strategy JSON configuration (required)") + flags.String("auth-key-env", "AML_API_KEY", "Environment variable holding the API key clients must send") + flags.String("consolidate", "off", "When to consolidate: off; on-add (during requests: full batches before an Add returns, the rest before the user's next Search); or background (in supervised workers, consolidation mode timelines or layered: an Add only writes, and a Search first folds the rest)") + flags.Bool("consolidate-on-add", false, "Older spelling of --consolidate on-add") + flags.Int("consolidation-workers", 4, "Users --consolidate background folds at once") + flags.String("admin-listen", "", "IP address and port of the admin listener (/metrics, /ready, /admin/status, /admin/v1/stores, /debug/pprof/): a loopback address, or a private one (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, fc00::/7) that needs --admin-token-env; empty for none") + flags.String("admin-token-env", "", "Environment variable holding the token (32 bytes at least) that every admin request must send as an Authorization: Bearer header; required for a private --admin-listen; empty for no token") + flags.Int("max-concurrent-adds", 0, "Most Adds served at once, 0 for no limit; past it an Add waits up to 30s for a slot, then answers 503 with Retry-After") + flags.Int("max-concurrent-searches", 0, "Most Searches served at once, 0 for no limit; past it a Search waits up to 30s for a slot, then answers 503 with Retry-After") + flags.String("log-format", "text", "Log format: text or json") + flags.Int("max-open-stores", 64, "Most user stores kept open at once") + flags.Duration("search-timeout", 10*time.Minute, "Longest memory read of a Search before it answers the raw ranking (1m to 18m)") + flags.String("provider-keys-file", "", "Absolute path of the account's provider key file, outside the data root: its DeepSeek and JEV keys are sent in place of the host's, and PUT and DELETE /admin/v1/provider-keys replace and revert them; a missing file holds none; empty for the host's keys alone") + flags.String("settings-file", "", "Absolute path of the instance's settings file, outside the data root: its DeepSeek and JEV budget caps (null for unlimited) replace the configuration's, and PUT and DELETE /admin/v1/budgets set and revert them; a missing file holds none; empty for the configuration's caps alone") + return command +} + +// amlServeOptions are aml-serve's checked flags. adminToken is what the +// variable --admin-token-env names holds, "" without the flag. +type amlServeOptions struct { + listen, adminListen, dataRoot, memoryConfig, keyEnv, logFormat string + adminToken, providerKeysFile, settingsFile string + consolidation aml.ConsolidationMode + workers, maxOpenStores, maxAdds, maxSearches int + searchTimeout time.Duration +} + +// amlServeFlags reads and checks aml-serve's flags, and the admin token's +// variable; it touches nothing else. +func amlServeFlags(cmd *cobra.Command) (amlServeOptions, error) { + flags := cmd.Flags() + var o amlServeOptions + o.listen, _ = flags.GetString("listen") + o.adminListen, _ = flags.GetString("admin-listen") + o.dataRoot, _ = flags.GetString("data-root") + o.memoryConfig, _ = flags.GetString("memory-config") + o.keyEnv, _ = flags.GetString("auth-key-env") + o.logFormat, _ = flags.GetString("log-format") + o.workers, _ = flags.GetInt("consolidation-workers") + o.maxOpenStores, _ = flags.GetInt("max-open-stores") + o.maxAdds, _ = flags.GetInt("max-concurrent-adds") + o.maxSearches, _ = flags.GetInt("max-concurrent-searches") + o.searchTimeout, _ = flags.GetDuration("search-timeout") + o.providerKeysFile, _ = flags.GetString("provider-keys-file") + o.settingsFile, _ = flags.GetString("settings-file") + var err error + if o.consolidation, err = amlConsolidation(cmd); err != nil { + return o, err + } + switch { + case !filepath.IsAbs(o.dataRoot): + return o, errors.New("aml-serve requires an absolute --data-root") + case o.maxOpenStores < 1: + return o, errors.New("--max-open-stores must be at least 1") + case o.searchTimeout < time.Minute || o.searchTimeout > 18*time.Minute: + // A Search may first flush for up to 10 minutes; both must end within + // the platform's 30-minute request timeout. + return o, errors.New("--search-timeout must be between 1m and 18m") + case o.keyEnv == "": + return o, errors.New("--auth-key-env must name an environment variable") + case o.workers < 1: + return o, errors.New("--consolidation-workers must be at least 1") + case o.maxAdds < 0: + return o, errors.New("--max-concurrent-adds must not be negative") + case o.maxSearches < 0: + return o, errors.New("--max-concurrent-searches must not be negative") + case o.logFormat != "text" && o.logFormat != "json": + return o, fmt.Errorf("--log-format must be text or json, not %q", o.logFormat) + case o.providerKeysFile != "" && !filepath.IsAbs(o.providerKeysFile): + return o, errors.New("--provider-keys-file must be an absolute path") + case o.settingsFile != "" && !filepath.IsAbs(o.settingsFile): + return o, errors.New("--settings-file must be an absolute path") + } + tokenEnv, _ := flags.GetString("admin-token-env") + if tokenEnv != "" && tokenEnv == o.keyEnv { + // The API key, which every client holds, would be the admin token. + return o, fmt.Errorf("--admin-token-env names $%s, which --auth-key-env names too: the admin token and the API key must be two secrets, in two variables", tokenEnv) + } + if o.adminToken, err = amlAdminToken(tokenEnv); err != nil { + return o, err + } + if err := checkAdminListen(o.adminListen, o.adminToken != ""); err != nil { + return o, err + } + return o, nil +} + +// tokenListener is a listener that the bind rule (checkListen) and the +// token rule (listenToken) guard, as their errors name it: its address +// flag, an example port, its token's flag, what the token is called, and +// what anyone who reaches the listener without one could do. aml-serve's +// admin listener and aml-inspect's listener follow the same rules. +type tokenListener struct { + listen, port, tokenEnv, token, exposes string +} + +var ( + adminListener = tokenListener{listen: "--admin-listen", port: "8091", tokenEnv: "--admin-token-env", token: "admin token", exposes: "profile the process and read its state"} + inspectListener = tokenListener{listen: "--listen", port: "8092", tokenEnv: "--token-env", token: "inspect token", exposes: "read every user's records"} +) + +// amlAdminToken is the admin token the environment variable name holds +// (listenToken). +func amlAdminToken(name string) (string, error) { return listenToken(adminListener, name) } + +// listenToken is the token the environment variable name holds, "" when +// name is empty (no token). A variable that is unset, empty or shorter than +// aml.MinAdminTokenLength bytes is an error that names the flag and the +// variable and says nothing of what it holds. +func listenToken(l tokenListener, name string) (string, error) { + if name == "" { + return "", nil + } + token := os.Getenv(name) + switch { + case token == "": + return "", fmt.Errorf("%s names $%s, which is unset or empty: it must hold the %s, %d bytes at least", l.tokenEnv, name, l.token, aml.MinAdminTokenLength) + case len(token) < aml.MinAdminTokenLength: + return "", fmt.Errorf("%s names $%s, which holds fewer than %d bytes: the %s must hold %d bytes at least", l.tokenEnv, name, aml.MinAdminTokenLength, l.token, aml.MinAdminTokenLength) + } + return token, nil +} + +// amlConsolidation is the consolidation mode --consolidate names, or its +// older spelling --consolidate-on-add; giving both is an error. +func amlConsolidation(cmd *cobra.Command) (aml.ConsolidationMode, error) { + flags := cmd.Flags() + if flags.Changed("consolidate-on-add") { + if flags.Changed("consolidate") { + return "", errors.New("--consolidate-on-add is the older spelling of --consolidate on-add: give one of them") + } + if onAdd, _ := flags.GetBool("consolidate-on-add"); onAdd { + return aml.ConsolidationOnAdd, nil + } + return aml.ConsolidationOff, nil + } + switch value, _ := flags.GetString("consolidate"); value { + case "off": + return aml.ConsolidationOff, nil + case "on-add": + return aml.ConsolidationOnAdd, nil + case "background": + return aml.ConsolidationBackground, nil + default: + return "", fmt.Errorf("--consolidate must be off, on-add or background, not %q", value) + } +} + +// checkAdminListen checks the address of the admin listener, "" for none +// (checkListen). +func checkAdminListen(addr string, tokenSet bool) error { + return checkListen(adminListener, addr, tokenSet) +} + +// checkListen checks the address of a listener, "" for none. Anyone who +// reaches the listener can do what l.exposes says, so it is a loopback +// address, which only this machine reaches, or a private one (netip's +// IsPrivate: 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 and fc00::/7) when +// its token is set, which only a client that holds it passes. Every other +// address is refused, the token or not: unspecified, link-local, shared +// (100.64.0.0/10) and public ones. +func checkListen(l tokenListener, addr string, tokenSet bool) error { + if addr == "" { + return nil + } + ip, ok := adminIP(addr) + if !ok { + return fmt.Errorf("%s must be an IP address and a port, such as 127.0.0.1:%s or 10.0.0.5:%s; a host name is not accepted, as what it resolves to is not the command's to check; not %q", l.listen, l.port, l.port, addr) + } + switch { + case ip.IsLoopback(): + return nil + case ip.IsPrivate() && tokenSet: + return nil + case ip.IsPrivate(): + return fmt.Errorf("%s %q is a private address, which needs %s: without a token anyone who reaches the listener can %s", l.listen, addr, l.tokenEnv, l.exposes) + } + return fmt.Errorf("%s must be a loopback IP address, or a private one (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, fc00::/7) with %s, not an unspecified, link-local, shared or public one; not %q", l.listen, l.tokenEnv, addr) +} + +// adminIP is the IP address of addr, which must be an address literal and a +// port, such as 127.0.0.1:8091 or [fd00::1]:8091. A host name, a missing +// port and a zone ([fe80::1%eth0]) are not accepted. An IPv4 address spelled +// as an IPv6 one is the IPv4 address. +func adminIP(addr string) (netip.Addr, bool) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return netip.Addr{}, false + } + if _, err := strconv.ParseUint(port, 10, 16); err != nil { + return netip.Addr{}, false + } + ip, err := netip.ParseAddr(host) + if err != nil || ip.Zone() != "" { + return netip.Addr{}, false + } + return ip.Unmap(), true +} + +// amlLogger is the logger of --log-format, to w. +func amlLogger(format string, w io.Writer) *slog.Logger { + if format == "json" { + return slog.New(slog.NewJSONHandler(w, nil)) + } + return slog.New(slog.NewTextHandler(w, nil)) +} + +// runAMLServe checks the flags (the admin token among them), the API key, the +// memory configuration and systemd's watchdog setting, takes the data root's +// lock (before anything opens a file in the data root or the budget file), +// builds the server, settles what a previous process left held in the budget +// file, binds the listeners and runs the process (aml_process.go) until +// SIGINT or SIGTERM. Its error, the exit status 1, also reports a shutdown +// that had to cut requests, consolidation jobs or model requests. +func runAMLServe(cmd *cobra.Command, _ []string) error { + if readOnly { + return errors.New("aml-serve writes user stores and cannot run with --readonly") + } + options, err := amlServeFlags(cmd) + if err != nil { + return err + } + key := os.Getenv(options.keyEnv) + if key == "" { + return fmt.Errorf("aml-serve requires a nonempty API key in $%s", options.keyEnv) + } + config, err := readMemoryConfig(options.memoryConfig) + if err != nil { + return err + } + // With WatchdogSec, systemd sets WATCHDOG_USEC (and WATCHDOG_PID); they + // stay set, as NOTIFY_SOCKET does, for the notifications to come. + watchdog, err := daemon.SdWatchdogEnabled(false) + if err != nil { + return fmt.Errorf("systemd watchdog: %w", err) + } + if watchdog > 0 && watchdog < amlMinWatchdog { + // The liveness probe reads a heartbeat that ticks every second. + return fmt.Errorf("systemd's watchdog interval %s is below %s: set WatchdogSec to at least %s", watchdog, amlMinWatchdog, amlMinWatchdog) + } + log := amlLogger(options.logFormat, cmd.ErrOrStderr()) + ctx, stopSignals := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer stopSignals() + lock, err := lockDataRoot(options.dataRoot) + if err != nil { + return err + } + defer lock.Unlock() + server, err := aml.NewServer(aml.ServerConfig{ + DataRoot: options.dataRoot, MemoryConfig: config, AuthKey: key, + Consolidation: options.consolidation, ConsolidationWorkers: options.workers, + MaxOpenStores: options.maxOpenStores, SearchTimeout: options.searchTimeout, + MaxConcurrentAdds: options.maxAdds, MaxConcurrentSearches: options.maxSearches, + Logger: log, AdminToken: options.adminToken, Version: version, ProviderKeysFile: options.providerKeysFile, + SettingsFile: options.settingsFile, + }) + if err != nil { + return err + } + // Under the data root's lock and before anything is served, so that the + // instance's budget file has no other writer: what a previous process + // left held there is settled (R90). A failure leaves it held, as before; + // the next start tries again. + if err := server.SettleHolds(ctx); err != nil { + log.Error(fmt.Sprintf("aml-serve: %v; it stays held until a start settles it", err)) + } + process, err := newAMLProcess(server, options, log, watchdog) + if err != nil { + // Nothing has run: no request or model call is in flight. + server.Close() + return err + } + return process.serve(ctx, stopSignals) +} diff --git a/cmd/memory/aml_admin_bind_test.go b/cmd/memory/aml_admin_bind_test.go new file mode 100644 index 000000000..dd63b9a90 --- /dev/null +++ b/cmd/memory/aml_admin_bind_test.go @@ -0,0 +1,93 @@ +package memory + +import ( + "net" + "net/http" + "slices" + "strings" + "testing" + "time" +) + +// shortAdminRetry makes aml-serve bind its admin listener again every +// interval until the test ends. +func shortAdminRetry(t *testing.T, interval time.Duration) { + t.Helper() + previous := amlAdminRetry + amlAdminRetry = interval + t.Cleanup(func() { amlAdminRetry = previous }) +} + +// R91: the data plane never depends on the admin bind. An --admin-listen +// address that cannot be bound when aml-serve starts (wg0 not up yet; here, +// a port another listener holds) does not keep the public listener from +// serving: aml-serve tells systemd READY=1, serves Adds, logs the failure +// once, and binds the address again every amlAdminRetry until it can, then +// serves the admin endpoints there and says so. It stops cleanly. +func TestAMLServeServesBeforeItsAdminListenerBinds(t *testing.T) { + shortAdminRetry(t, 50*time.Millisecond) + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + held, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer held.Close() + adminAddr := held.Addr().String() + run := startAMLServe(t, amlTestConfig(t, endpoint.URL, nil), "--admin-listen", adminAddr) + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + public := run.address(t, false) + if code := get(t, public+"/health"); code != http.StatusOK { + t.Fatal("public /health", code) + } + if code, body := run.post(t, "/add", noteAdd("r1", "u1")); code != http.StatusOK { + t.Fatal(code, body) + } + // It has tried again several times by now, and said so once. + time.Sleep(300 * time.Millisecond) + if log := run.log.String(); strings.Count(log, "--admin-listen") != 1 || !strings.Contains(log, "serving without the admin listener and binding it again every 50ms") || strings.Contains(log, "admin listening on") { + t.Fatal(log) + } + held.Close() + if admin := run.address(t, true); admin != "http://"+adminAddr { + t.Fatal("the admin listener bound", admin, "want", adminAddr) + } + if code := get(t, "http://"+adminAddr+"/admin/status"); code != http.StatusOK { + t.Fatal("admin /admin/status", code) + } + if err := run.stop(t); err != nil { + t.Fatal(err, run.log.String()) + } + if got := notified.all(); !slices.Equal(got, []string{"READY=1", "STOPPING=1"}) { + t.Fatal(got) + } +} + +// R91: a shutdown while the admin listener is still unbound stops its +// retries with the rest: the command returns, cleanly, and the address is +// never bound. +func TestAMLServeStopsWhileItsAdminListenerIsUnbound(t *testing.T) { + shortAdminRetry(t, 20*time.Millisecond) + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + held, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer held.Close() + run := startAMLServe(t, amlTestConfig(t, endpoint.URL, nil), "--admin-listen", held.Addr().String()) + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + if err := run.stop(t); err != nil { + t.Fatal(err, run.log.String()) + } + held.Close() + time.Sleep(100 * time.Millisecond) // a retry left running would bind now + probe, err := net.Listen("tcp", held.Addr().String()) + if err != nil { + t.Fatal("the address was bound after the command returned:", err) + } + probe.Close() + if strings.Contains(run.log.String(), "admin listening on") { + t.Fatal(run.log.String()) + } +} diff --git a/cmd/memory/aml_holds_test.go b/cmd/memory/aml_holds_test.go new file mode 100644 index 000000000..709597ad5 --- /dev/null +++ b/cmd/memory/aml_holds_test.go @@ -0,0 +1,308 @@ +package memory + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/mnemon-dev/mnemon/internal/memoryagent" + "github.com/mnemon-dev/mnemon/internal/records" +) + +// These tests hold aml-serve and aml-inspect to ruling R90: a restart holds +// nothing for good. A stop cuts the model requests still in flight at the end +// of its windows, which settle before the process exits (aml_test.go's +// TestShutdownCutsAbandonedModelRequests), and a start settles what a crash +// left held in the instance's own budget file, under the instance's lock. + +// leftHold is a reservation a process that ended left held: its kind, what +// it holds, and the input estimate its attempt recorded (cost and tokens). +type leftHold struct { + kind string + reserved float64 + estimate float64 + estimated int +} + +// leaveHolds writes holds into the budget file at path as a process that +// crashed with requests in flight leaves them. +func leaveHolds(t *testing.T, path string, holds ...leftHold) { + t.Helper() + budget, err := memoryagent.OpenBudget(path) + if err != nil { + t.Fatal(err) + } + budget.Close() + db, err := sql.Open("sqlite", records.SQLiteURI(path, nil)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for i, h := range holds { + for _, statement := range []struct { + query string + args []any + }{ + {`INSERT INTO accounts(kind,cap) VALUES(?,100) ON CONFLICT(kind) DO NOTHING`, []any{h.kind}}, + {`INSERT INTO attempts(id,kind,request_hash,reserved,estimate,estimate_tokens) VALUES(?,?,?,?,?,?)`, []any{fmt.Sprintf("left-%d", i), h.kind, "h", h.reserved, h.estimate, h.estimated}}, + {`UPDATE accounts SET held=held+? WHERE kind=?`, []any{h.reserved, h.kind}}, + } { + if _, err := db.Exec(statement.query, statement.args...); err != nil { + t.Fatal(statement.query, err) + } + } + } +} + +// budgetFileAccount is what the budget file at path says of kind's account: +// held and spent. +func budgetFileAccount(t *testing.T, path, kind string) (held, spent float64) { + t.Helper() + budget, err := memoryagent.OpenBudget(path) + if err != nil { + t.Fatal(err) + } + defer budget.Close() + accounts, err := budget.Accounts(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, account := range accounts { + if account.Kind == kind { + return account.Held, account.Spent + } + } + t.Fatalf("no %s account in %v", kind, accounts) + return 0, 0 +} + +// budgetAccount is budgetFileAccount for the budget file of the memory +// configuration at configPath. +func budgetAccount(t *testing.T, configPath, kind string) (held, spent float64) { + t.Helper() + config, err := readMemoryConfig(configPath) + if err != nil { + t.Fatal(err) + } + return budgetFileAccount(t, config.BudgetFile, kind) +} + +// near reports whether two amounts agree to 1e-9. +func near(a, b float64) bool { return a-b < 1e-9 && b-a < 1e-9 } + +// R90: the holds a previous process left in the instance's budget file, its +// requests cut by a crash, are settled at the input estimates their attempts +// recorded when aml-serve starts, under the data root's lock and before it +// listens, and one line says what it settled, by kind. A start with nothing +// held says nothing of it. +func TestAMLServeSettlesTheHoldsAPreviousProcessLeft(t *testing.T) { + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + configPath := amlTestConfig(t, endpoint.URL, nil) + config, err := readMemoryConfig(configPath) + if err != nil { + t.Fatal(err) + } + leaveHolds(t, config.BudgetFile, + leftHold{"deepseek", 0.25, 0.0021, 1050}, + leftHold{"deepseek", 0.25, 0.0009, 450}, + leftHold{"jev", 0.01, 0.000042, 1000}) + run := startAMLServe(t, configPath) + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + if err := run.stop(t); err != nil { + t.Fatal(err, run.log.String()) + } + log := run.log.String() + settled := strings.Index(log, "settled 3 budget reservations a previous process left held, at their input estimates: deepseek 2 (held 0.5000, settled 0.0030), jev 1 (held 0.0100, settled 0.0000)") + if listening := strings.Index(log, "aml-serve listening on"); settled < 0 || listening < settled || strings.Count(log, "budget reservations") != 1 { + t.Fatal(log) + } + if held, spent := budgetFileAccount(t, config.BudgetFile, "deepseek"); held != 0 || !near(spent, 0.003) { + t.Fatal("deepseek held", held, "spent", spent) + } + if held, spent := budgetFileAccount(t, config.BudgetFile, "jev"); held != 0 || !near(spent, 0.000042) { + t.Fatal("jev held", held, "spent", spent) + } + + // Nothing is held now: the next start says nothing of it. + again := startAMLServe(t, configPath) + waitFor(t, "READY=1 again", func() bool { + ready := 0 + for _, state := range notified.all() { + if state == "READY=1" { + ready++ + } + } + return ready == 2 + }) + if err := again.stop(t); err != nil { + t.Fatal(err) + } + if strings.Contains(again.log.String(), "budget reservations") { + t.Fatal(again.log.String()) + } +} + +// shortInspectGrace makes aml-inspect's shutdown windows short until the test +// ends. +func shortInspectGrace(t *testing.T, grace, cancel time.Duration) { + t.Helper() + previousGrace, previousCancel := inspectShutdownGrace, inspectCancelGrace + inspectShutdownGrace, inspectCancelGrace = grace, cancel + t.Cleanup(func() { inspectShutdownGrace, inspectCancelGrace = previousGrace, previousCancel }) +} + +// amlInspectRun is an aml-inspect command the test runs until it stops it. +type amlInspectRun struct { + log *syncLog + stop context.CancelFunc + done chan error + addr string +} + +// startAMLInspect runs aml-inspect on a loopback port of its own, without a +// token, over root with configPath, its full searches billed to debug; it +// returns once the command listens, or with the command's error. +func startAMLInspect(t *testing.T, root, configPath, debug string) (*amlInspectRun, error) { + t.Helper() + run := &amlInspectRun{log: &syncLog{}, done: make(chan error, 1)} + command := newAMLInspectCommand() + command.SetArgs([]string{"--listen", "127.0.0.1:0", "--data-root", root, "--memory-config", configPath, + "--debug-budget-file", debug, "--debug-caps", "deepseek=5,jev=1", "--log-format", "json"}) + command.SilenceUsage, command.SilenceErrors = true, true + command.SetOut(io.Discard) + command.SetErr(run.log) + ctx, stop := context.WithCancel(context.Background()) + run.stop = stop + go func() { run.done <- command.ExecuteContext(ctx) }() + t.Cleanup(func() { + stop() + select { + case <-run.done: + case <-time.After(30 * time.Second): + } + }) + for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(5 * time.Millisecond) { + if match := inspectListening.FindStringSubmatch(run.log.String()); match != nil { + run.addr = match[1] + return run, nil + } + select { + case err := <-run.done: + run.done <- err + return run, err + default: + } + } + t.Fatal("aml-inspect did not listen:", run.log.String()) + return nil, nil +} + +// end stops the command as SIGTERM does and returns its error, and how long +// it took to return. +func (r *amlInspectRun) end(t *testing.T) (error, time.Duration) { + t.Helper() + start := time.Now() + r.stop() + select { + case err := <-r.done: + r.done <- err + return err, time.Since(start) + case <-time.After(30 * time.Second): + t.Fatal("aml-inspect did not stop") + return nil, 0 + } +} + +// R90 for the inspect service: it takes its debug budget's lock, settles the +// holds a previous run left there at their recorded estimates before it +// listens, and says so in one line; a second aml-inspect on the same debug +// budget refuses to start while the first runs. +func TestAMLInspectSettlesTheHoldsAPreviousRunLeft(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "users"), 0o700); err != nil { + t.Fatal(err) + } + configPath := amlTestConfig(t, "http://127.0.0.1:1", nil) + debug := filepath.Join(t.TempDir(), "debug-budget.db") + leaveHolds(t, debug, leftHold{"deepseek", 0.25, 0.004, 2000}) + first, err := startAMLInspect(t, root, configPath, debug) + if err != nil { + t.Fatal(err) + } + log := first.log.String() + settled := strings.Index(log, "settled 1 budget reservation a previous process left held, at its input estimate: deepseek 1 (held 0.2500, settled 0.0040)") + if listening := strings.Index(log, "aml-inspect listening on"); settled < 0 || listening < settled { + t.Fatal(log) + } + if _, err := startAMLInspect(t, root, configPath, debug); err == nil || err.Error() != "another aml-inspect holds "+debug+".lock" { + t.Fatal("a second aml-inspect on the same debug budget:", err) + } + if err, _ := first.end(t); err != nil { + t.Fatal(err, first.log.String()) + } + if held, spent := budgetFileAccount(t, debug, "deepseek"); held != 0 || !near(spent, 0.004) { + t.Fatal("held", held, "spent", spent) + } +} + +// Minor 2 and R90: aml-inspect's shutdown is bounded as aml-serve's is. +// Closing its models waits for their requests in flight, those of a full +// search whose planner attempts were abandoned included, until the end of +// its windows, then cuts them: they settle in the debug budget, and the +// command returns within a few seconds of the windows (TimeoutStopSec=45 +// leaves it more), saying what it cut. +func TestAMLInspectShutdownCutsAbandonedModelRequests(t *testing.T) { + shortInspectGrace(t, 100*time.Millisecond, 400*time.Millisecond) + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + // A planner attempt stops waiting after one second and is not retried; + // its request runs on toward the minute's hard timeout. + configPath := amlTestConfig(t, endpoint.URL, func(config *memoryagent.Config) { + config.Planner.TimeoutSeconds, config.Planner.AttemptTimeoutSeconds, config.Planner.MaxAttempts = 60, 1, 1 + }) + // An aml-serve run gives the data root a user's store. + serve := startAMLServe(t, configPath) + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + if code, body := serve.post(t, "/add", noteAdd("r1", "u1")); code != http.StatusOK { + t.Fatal(code, body) + } + if err := serve.stop(t); err != nil { + t.Fatal(err) + } + debug := filepath.Join(t.TempDir(), "debug-budget.db") + run, err := startAMLInspect(t, serve.root, configPath, debug) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(map[string]any{"user_id": "u1", "query": "When did Caroline go to the LGBTQ support group?", "mode": "full"}) + response, err := (&http.Client{Timeout: 30 * time.Second}).Post("http://"+run.addr+"/v1/search", "application/json", bytes.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + io.Copy(io.Discard, response.Body) + response.Body.Close() + if n := endpoint.requests.Load(); n < 1 { + t.Fatal("planner requests", n) + } + err, took := run.end(t) + if took > 5*time.Second { + t.Fatal("the shutdown waited for the abandoned requests", took) + } + if err == nil || !strings.Contains(err.Error(), "deepseek: cut ") || !strings.Contains(err.Error(), "still in flight at the close's deadline: their reservations settled at their input estimates") { + t.Fatal(err, run.log.String()) + } + if held, spent := budgetFileAccount(t, debug, "deepseek"); held != 0 || spent <= 0 { + t.Fatal("debug budget deepseek held", held, "spent", spent) + } +} diff --git a/cmd/memory/aml_inspect.go b/cmd/memory/aml_inspect.go new file mode 100644 index 000000000..51892cb49 --- /dev/null +++ b/cmd/memory/aml_inspect.go @@ -0,0 +1,264 @@ +package memory + +import ( + "context" + "errors" + "fmt" + "log/slog" + "math" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/gofrs/flock" + "github.com/mnemon-dev/mnemon/internal/aml" + "github.com/mnemon-dev/mnemon/internal/memoryagent" + "github.com/spf13/cobra" +) + +// newAMLInspectCommand is `mnemon memory-agent aml-inspect`: the inspect +// service of one engine instance, the memory cloud console's read-only +// retrieval backend (docs/inspect-api.md). It runs beside aml-serve, never +// in its process, and reads its data root without writing anything there. +func newAMLInspectCommand() *cobra.Command { + command := &cobra.Command{ + Use: "aml-inspect", + Short: "Serve side-effect-free retrieval over an aml-serve data root, for the console", + Args: cobra.NoArgs, + RunE: runAMLInspect, + } + flags := command.Flags() + flags.String("data-root", "", "Absolute aml-serve data root to read (required); nothing in it is ever written") + flags.String("memory-config", "", "Absolute path of the instance's memory-strategy JSON configuration (required): the read's settings and models") + flags.String("listen", "127.0.0.1:8092", "IP address and port to listen on: a loopback address, or a private one (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, fc00::/7) that needs --token-env") + flags.String("token-env", "", "Environment variable holding the token (32 bytes at least) that every request must send as an Authorization: Bearer header; required for a private --listen; empty for no token") + flags.String("debug-budget-file", "", "Absolute budget file full searches reserve and settle in, never the configuration's budgetFile; empty for no full searches") + flags.String("debug-caps", "", "The debug budget's caps by kind, such as deepseek=5,jev=1 (DeepSeek in CNY, JEV in USD); full searches are refused while none is set") + flags.Int("max-concurrent", 2, "Most requests reading stores at once; past it a request answers 503 with Retry-After") + flags.String("log-format", "text", "Log format: text or json") + flags.String("provider-keys-file", "", "Absolute path of the instance's provider key file (aml-serve's --provider-keys-file), read at the start and before each full search, never written: its DeepSeek and JEV keys are sent in place of the host's; empty for the host's keys alone") + flags.String("settings-file", "", "Absolute path of the service's own settings file, outside the data root: its debug caps (null for unlimited) replace those of --debug-caps, and PUT and DELETE /v1/debug-caps set and revert them; needs --debug-budget-file; a missing file holds none; empty for --debug-caps alone") + return command +} + +// amlInspectOptions are aml-inspect's checked flags. token is what the +// variable --token-env names holds, "" without the flag. +type amlInspectOptions struct { + dataRoot, memoryConfig, listen, logFormat, debugBudget, token string + providerKeysFile, settingsFile string + caps map[string]float64 + maxConcurrent int +} + +// amlInspectFlags reads and checks aml-inspect's flags, and the token's +// variable; it touches nothing else. --listen and --token-env follow the +// admin listener's rules (checkListen, listenToken). +func amlInspectFlags(cmd *cobra.Command) (amlInspectOptions, error) { + flags := cmd.Flags() + var o amlInspectOptions + o.dataRoot, _ = flags.GetString("data-root") + o.memoryConfig, _ = flags.GetString("memory-config") + o.listen, _ = flags.GetString("listen") + o.logFormat, _ = flags.GetString("log-format") + o.debugBudget, _ = flags.GetString("debug-budget-file") + o.maxConcurrent, _ = flags.GetInt("max-concurrent") + o.providerKeysFile, _ = flags.GetString("provider-keys-file") + o.settingsFile, _ = flags.GetString("settings-file") + caps, _ := flags.GetString("debug-caps") + switch { + case !filepath.IsAbs(o.dataRoot): + return o, errors.New("aml-inspect requires an absolute --data-root") + case o.maxConcurrent < 1: + return o, errors.New("--max-concurrent must be at least 1") + case o.logFormat != "text" && o.logFormat != "json": + return o, fmt.Errorf("--log-format must be text or json, not %q", o.logFormat) + case o.debugBudget != "" && !filepath.IsAbs(o.debugBudget): + return o, errors.New("--debug-budget-file must be an absolute path") + case caps != "" && o.debugBudget == "": + return o, errors.New("--debug-caps needs --debug-budget-file") + case o.providerKeysFile != "" && !filepath.IsAbs(o.providerKeysFile): + return o, errors.New("--provider-keys-file must be an absolute path") + case o.settingsFile != "" && !filepath.IsAbs(o.settingsFile): + return o, errors.New("--settings-file must be an absolute path") + case o.settingsFile != "" && o.debugBudget == "": + return o, errors.New("--settings-file needs --debug-budget-file") + } + var err error + if o.caps, err = debugCaps(caps); err != nil { + return o, err + } + tokenEnv, _ := flags.GetString("token-env") + if o.token, err = listenToken(inspectListener, tokenEnv); err != nil { + return o, err + } + if o.listen == "" { + // Unlike the admin listener, the service always listens. + return o, errors.New("--listen must be an IP address and a port, such as 127.0.0.1:8092 or 10.0.0.5:8092: aml-inspect always listens") + } + return o, checkListen(inspectListener, o.listen, o.token != "") +} + +// debugCaps reads --debug-caps: kind=amount pairs separated by commas, each +// kind once; "" is none. +func debugCaps(value string) (map[string]float64, error) { + caps := map[string]float64{} + if value == "" { + return caps, nil + } + for _, pair := range strings.Split(value, ",") { + kind, amount, ok := strings.Cut(pair, "=") + cap, err := strconv.ParseFloat(amount, 64) + if !ok || err != nil || math.IsNaN(cap) || math.IsInf(cap, 0) || cap < 0 { + return nil, fmt.Errorf("--debug-caps must be kind=amount pairs, such as deepseek=5,jev=1, not %q", pair) + } + if _, paid := memoryagent.PaidKinds()[kind]; !paid { + return nil, fmt.Errorf("--debug-caps kind %q: the kinds are %s", kind, aml.PaidKindNames()) + } + if _, dup := caps[kind]; dup { + return nil, fmt.Errorf("--debug-caps names %s twice", kind) + } + caps[kind] = cap + } + return caps, nil +} + +// inspectShutdownGrace is how long the requests in flight have to finish +// once the inspect service stops; inspectCancelGrace, how long the requests +// canceled then have to end; inspectSettleGrace, how long the model requests +// that the models' close cuts at the end of both windows have to settle +// (R90). 30 s + 5 s + 5 s stays within the unit's TimeoutStopSec=45. Tests +// shorten the first two. +var ( + inspectShutdownGrace = 30 * time.Second + inspectCancelGrace = 5 * time.Second + inspectSettleGrace = 5 * time.Second +) + +// lockDebugBudget takes the debug budget's lock, an exclusive lock on +// .lock, a file of its own beside it: one aml-inspect per +// debug budget, so that what the service finds held there when it starts is +// a previous run's, which it settles (R90). The kernel releases the lock +// when the process ends, however it ends; while another process holds it, +// the command refuses to run. +func lockDebugBudget(path string) (*flock.Flock, error) { + lock := flock.New(path + ".lock") + locked, err := lock.TryLock() + if err != nil { + return nil, fmt.Errorf("lock %s: %w", lock.Path(), err) + } + if !locked { + return nil, fmt.Errorf("another aml-inspect holds %s", lock.Path()) + } + return lock, nil +} + +// runAMLInspect checks the flags (the token among them) and the memory +// configuration, builds the inspect service (which checks the data root and +// the debug budget file, and opens the latter), takes the debug budget's +// lock and settles what a previous run left held there, binds the listener +// and serves until SIGINT or SIGTERM. It never takes the data root's lock or +// opens a file in the data root but the stores it reads for a request. +func runAMLInspect(cmd *cobra.Command, _ []string) error { + options, err := amlInspectFlags(cmd) + if err != nil { + return err + } + config, err := readMemoryConfig(options.memoryConfig) + if err != nil { + return err + } + log := amlLogger(options.logFormat, cmd.ErrOrStderr()) + ctx, stopSignals := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer stopSignals() + inspector, err := aml.NewInspector(aml.InspectConfig{ + DataRoot: options.dataRoot, MemoryConfig: config, Token: options.token, + DebugBudgetFile: options.debugBudget, DebugCaps: options.caps, MaxConcurrent: options.maxConcurrent, + Logger: log, Version: version, ProviderKeysFile: options.providerKeysFile, SettingsFile: options.settingsFile, + }) + if err != nil { + return err + } + if options.debugBudget != "" { + lock, err := lockDebugBudget(filepath.Clean(options.debugBudget)) + if err != nil { + inspector.Close() + return err + } + defer lock.Unlock() + if err := inspector.SettleHolds(ctx); err != nil { + log.Error(fmt.Sprintf("aml-inspect: %v; it stays held until a start settles it", err)) + } + } + listener, err := net.Listen("tcp", options.listen) + if err != nil { + inspector.Close() + return err + } + log.Info(fmt.Sprintf("aml-inspect listening on %s", listener.Addr())) + return serveInspect(ctx, stopSignals, inspector, listener, log) +} + +// serveInspect serves the inspect service on listener until ctx ends or the +// server fails, then shuts down: the requests in flight have +// inspectShutdownGrace to finish, those still running are canceled and have +// inspectCancelGrace more, then the connections close; and the models and +// the debug budget close within what is left of both windows, as aml-serve's +// do (the final review's Minor 2, R90): the models' requests in flight, +// abandoned ones included, are cut at the windows' end and settle in the +// debug budget within inspectSettleGrace more, after which the command +// returns all the same. The serving goroutine is this function's: it is +// waited for before it returns. +func serveInspect(ctx context.Context, stopSignals func(), inspector *aml.Inspector, listener net.Listener, log *slog.Logger) error { + server := inspector.HTTPServer() + served := make(chan error, 1) + go func() { served <- server.Serve(listener) }() + var failed error + select { + case <-ctx.Done(): + log.Info("aml-inspect: shutting down") + case failed = <-served: + log.Error(fmt.Sprintf("aml-inspect: %v; shutting down", failed)) + } + stopSignals() + graceEnd := time.Now().Add(inspectShutdownGrace) + end := graceEnd.Add(inspectCancelGrace) + grace, endGrace := context.WithDeadline(context.Background(), graceEnd) + defer endGrace() + var cut []error + if err := server.Shutdown(grace); errors.Is(err, context.DeadlineExceeded) { + log.Warn(fmt.Sprintf("aml-inspect: the %s grace has ended with requests still running; canceling them", inspectShutdownGrace)) + inspector.CancelRequests() + final, endFinal := context.WithDeadline(context.Background(), end) + if server.Shutdown(final) != nil { + server.Close() + } + endFinal() + cut = append(cut, fmt.Errorf("canceled at the end of the %s grace: requests still running", inspectShutdownGrace)) + } + if failed == nil { + if err := <-served; !errors.Is(err, http.ErrServerClosed) { + failed = err + } + } + closing, endClosing := context.WithDeadline(context.Background(), end) + defer endClosing() + var closeErr error + switch { + case !closeBy(func() error { closeErr = inspector.CloseWithin(closing); return closeErr }, end.Add(inspectSettleGrace)): + log.Error(fmt.Sprintf("aml-inspect: closing the models did not finish %s after the end of the shutdown; exiting while model requests settle: their debug budget reservations stay held until the next start settles them", inspectSettleGrace)) + cut = append(cut, fmt.Errorf("closing the models did not finish %s after the end of the shutdown: the model requests still settling keep their debug budget reservations held until the next start settles them", inspectSettleGrace)) + case closeErr != nil: + log.Warn(fmt.Sprintf("aml-inspect: %v", closeErr)) + cut = append(cut, closeErr) + } + if errors.Is(failed, http.ErrServerClosed) { + failed = nil + } + return errors.Join(append([]error{failed}, cut...)...) +} diff --git a/cmd/memory/aml_inspect_test.go b/cmd/memory/aml_inspect_test.go new file mode 100644 index 000000000..eb765f1c3 --- /dev/null +++ b/cmd/memory/aml_inspect_test.go @@ -0,0 +1,241 @@ +package memory + +import ( + "context" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +func TestAMLInspectIsAMemoryAgentSubcommand(t *testing.T) { + command, _, err := rootCmd.Find([]string{"memory-agent", "aml-inspect"}) + if err != nil || command.Name() != "aml-inspect" || command.Parent().Name() != "memory-agent" { + t.Fatal(command, err) + } + for flag, value := range map[string]string{ + "data-root": "", "memory-config": "", "listen": "127.0.0.1:8092", "token-env": "", "debug-budget-file": "", + "debug-caps": "", "max-concurrent": "2", "log-format": "text", + } { + if f := command.Flags().Lookup(flag); f == nil || f.DefValue != value { + t.Fatal(flag, f) + } + } +} + +// inspectFlags parses args, after an absolute --data-root and +// --memory-config, as aml-inspect's flags. +func inspectFlags(t *testing.T, args ...string) (amlInspectOptions, error) { + t.Helper() + command := newAMLInspectCommand() + dir := t.TempDir() + if err := command.ParseFlags(append([]string{"--data-root", dir, "--memory-config", filepath.Join(dir, "memory.json")}, args...)); err != nil { + t.Fatal(err) + } + return amlInspectFlags(command) +} + +// aml-inspect's --listen and --token-env follow aml-serve's admin listener's +// rules (E1), with the same cases: a loopback address needs no token, a +// private one needs it, every other is refused; the token holds 32 +// bytes at least, and no error shows it. +func TestInspectAuthAndListen(t *testing.T) { + const variable = "AML_TEST_INSPECT_TOKEN" + token := strings.Repeat("t", 32) + wantError := func(t *testing.T, err error, parts ...string) { + t.Helper() + if err == nil { + t.Fatal("no error") + } + for _, part := range parts { + if !strings.Contains(err.Error(), part) { + t.Fatalf("%q lacks %q", err, part) + } + } + if strings.Contains(err.Error(), "admin") { + t.Fatalf("%q names the admin listener", err) + } + } + for _, addr := range loopbackListens { + t.Setenv(variable, "") + if options, err := inspectFlags(t, "--listen", addr); err != nil || options.listen != addr || options.token != "" { + t.Fatal(addr, err) + } + t.Setenv(variable, token) + if options, err := inspectFlags(t, "--listen", addr, "--token-env", variable); err != nil || options.token != token { + t.Fatal(addr, err) + } + } + for _, addr := range privateListens { + _, err := inspectFlags(t, "--listen", addr) + wantError(t, err, "--listen", "private", "--token-env", "read every user's records") + t.Setenv(variable, token) + if options, err := inspectFlags(t, "--listen", addr, "--token-env", variable); err != nil || options.listen != addr || options.token != token { + t.Fatal(addr, err) + } + } + for _, addr := range append([]string{""}, refusedListens...) { + _, err := inspectFlags(t, "--listen", addr) + wantError(t, err, "--listen must be") + t.Setenv(variable, token) + _, err = inspectFlags(t, "--listen", addr, "--token-env", variable) + wantError(t, err, "--listen must be") + } + const missing = "AML_TEST_NO_SUCH_INSPECT_TOKEN" + for _, addr := range []string{"127.0.0.1:8092", "10.0.0.5:8092"} { + t.Setenv(variable, "") + _, err := inspectFlags(t, "--listen", addr, "--token-env", variable) + wantError(t, err, "--token-env", variable, "unset or empty", "inspect token") + os.Unsetenv(missing) + _, err = inspectFlags(t, "--listen", addr, "--token-env", missing) + wantError(t, err, "--token-env", missing, "unset or empty") + short := strings.Repeat("s", 31) + t.Setenv(variable, short) + _, err = inspectFlags(t, "--listen", addr, "--token-env", variable) + wantError(t, err, "--token-env", variable, "32 bytes") + if strings.Contains(err.Error(), short) { + t.Fatal("the error shows the token:", err) + } + } +} + +func TestAMLInspectFlags(t *testing.T) { + // An absolute path on every system: "/d.db" is not one on Windows (R89). + debug := filepath.Join(t.TempDir(), "debug-budget.db") + options, err := inspectFlags(t, "--debug-budget-file", debug, "--debug-caps", "deepseek=5,jev=1.5", "--max-concurrent", "3", "--log-format", "json") + if err != nil || options.debugBudget != debug || options.caps["deepseek"] != 5 || options.caps["jev"] != 1.5 || + len(options.caps) != 2 || options.maxConcurrent != 3 || options.logFormat != "json" { + t.Fatalf("%+v %v", options, err) + } + for args, want := range map[string]string{ + "--debug-caps deepseek=5": "--debug-caps needs --debug-budget-file", + "--debug-budget-file debug.db": "--debug-budget-file must be an absolute path", + "--debug-budget-file DEBUG --debug-caps deepseek": `--debug-caps must be kind=amount pairs, such as deepseek=5,jev=1, not "deepseek"`, + "--debug-budget-file DEBUG --debug-caps deepseek=x": `--debug-caps must be kind=amount pairs, such as deepseek=5,jev=1, not "deepseek=x"`, + "--debug-budget-file DEBUG --debug-caps deepseek=-1": `--debug-caps must be kind=amount pairs, such as deepseek=5,jev=1, not "deepseek=-1"`, + "--debug-budget-file DEBUG --debug-caps jev=1,jev=2": `--debug-caps names jev twice`, + "--debug-budget-file DEBUG --debug-caps claude=1": `--debug-caps kind "claude": the kinds are deepseek, jev and openai`, + "--max-concurrent 0": "--max-concurrent must be at least 1", + "--log-format yaml": `--log-format must be text or json, not "yaml"`, + "--debug-budget-file DEBUG --debug-caps deepseek=5,,jev": `--debug-caps must be kind=amount pairs, such as deepseek=5,jev=1, not ""`, + } { + fields := strings.Fields(args) + for i, field := range fields { + if field == "DEBUG" { + fields[i] = debug + } + } + if _, err := inspectFlags(t, fields...); err == nil || err.Error() != want { + t.Errorf("%q: %v", args, err) + } + } + command := newAMLInspectCommand() + if err := command.ParseFlags([]string{"--data-root", "data", "--memory-config", "/m.json"}); err != nil { + t.Fatal(err) + } + if _, err := amlInspectFlags(command); err == nil || err.Error() != "aml-inspect requires an absolute --data-root" { + t.Fatal(err) + } +} + +var inspectListening = regexp.MustCompile(`aml-inspect listening on (127\.0\.0\.1:\d+)`) + +// The command serves over a data root it never writes until it is stopped, +// with its token on every route, and exits 0. +func TestAMLInspectServesUntilStopped(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "users"), 0o700); err != nil { + t.Fatal(err) + } + configPath := amlTestConfig(t, "http://127.0.0.1:1", nil) + token := strings.Repeat("k", 40) + t.Setenv("AML_TEST_INSPECT_TOKEN", token) + debug := filepath.Join(t.TempDir(), "debug-budget.db") + command := newAMLInspectCommand() + log := &syncLog{} + command.SetArgs([]string{"--listen", "127.0.0.1:0", "--token-env", "AML_TEST_INSPECT_TOKEN", "--data-root", root, "--memory-config", configPath, + "--debug-budget-file", debug, "--debug-caps", "deepseek=5,jev=1", "--log-format", "json"}) + command.SilenceUsage, command.SilenceErrors = true, true + command.SetOut(io.Discard) + command.SetErr(log) + ctx, stop := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- command.ExecuteContext(ctx) }() + var addr string + waitFor(t, "the listening line", func() bool { + if match := inspectListening.FindStringSubmatch(log.String()); match != nil { + addr = match[1] + } + return addr != "" + }) + get := func(path, auth string) (int, map[string]any) { + request, _ := http.NewRequest(http.MethodGet, "http://"+addr+path, nil) + if auth != "" { + request.Header.Set("Authorization", "Bearer "+auth) + } + response, err := (&http.Client{Timeout: 10 * time.Second}).Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var body map[string]any + json.NewDecoder(response.Body).Decode(&body) + return response.StatusCode, body + } + if status, _ := get("/healthz", ""); status != http.StatusUnauthorized { + t.Fatal(status) + } + if status, body := get("/healthz", token); status != http.StatusOK || body["ok"] != true { + t.Fatal(status, body) + } + if status, body := get("/v1/info", token); status != http.StatusOK || body["modes"].(map[string]any)["full"] != true || body["version"] != version { + t.Fatal(status, body) + } + stop() + select { + case err := <-done: + if err != nil { + t.Fatal(err, log.String()) + } + case <-time.After(30 * time.Second): + t.Fatal("aml-inspect did not stop") + } + if strings.Contains(log.String(), token) { + t.Fatal("the log holds the token") + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 1 { + t.Fatal("aml-inspect touched the data root:", entries, err) + } + if _, err := os.Stat(debug); err != nil { + t.Fatal("no debug budget file", err) + } +} + +// The command refuses a debug budget file that is the configuration's own, +// before it listens. +func TestAMLInspectRefusesTheInstanceBudget(t *testing.T) { + root := t.TempDir() + os.Mkdir(filepath.Join(root, "users"), 0o700) + configPath := amlTestConfig(t, "http://127.0.0.1:1", nil) + config, err := readMemoryConfig(configPath) + if err != nil { + t.Fatal(err) + } + command := newAMLInspectCommand() + command.SetArgs([]string{"--listen", "127.0.0.1:0", "--data-root", root, "--memory-config", configPath, "--debug-budget-file", config.BudgetFile, "--debug-caps", "deepseek=5,jev=1"}) + command.SilenceUsage, command.SilenceErrors = true, true + command.SetOut(io.Discard) + command.SetErr(io.Discard) + if err := command.Execute(); err == nil || !strings.Contains(err.Error(), "must differ from the configuration's budgetFile") { + t.Fatal(err) + } + if _, err := os.Stat(config.BudgetFile); !os.IsNotExist(err) { + t.Fatal("the refused command made the instance's budget file", err) + } +} diff --git a/cmd/memory/aml_process.go b/cmd/memory/aml_process.go new file mode 100644 index 000000000..24b737242 --- /dev/null +++ b/cmd/memory/aml_process.go @@ -0,0 +1,549 @@ +package memory + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "github.com/coreos/go-systemd/v22/daemon" + "github.com/gofrs/flock" + "github.com/mnemon-dev/mnemon/internal/aml" + "golang.org/x/sync/errgroup" +) + +// The aml-serve process: its long-lived components run under one supervisor, +// which starts them in order and stops them in phases. +// +// Startup, once the flags are checked, the data root locked and the server +// built (runAMLServe): the public listener is bound, and the admin listener +// too when its address can be (see below); the server's background work +// (aml.Server.Run) starts, and is ready once its startup scan has queued +// every user and its heartbeat ticks; then the admin server, the public +// server and systemd's watchdog start, each once the one before is ready; +// then systemd is told READY=1. The data plane never depends on the admin +// bind (ruling R91): an admin listener that cannot bind (wg0 not up yet) is +// logged once, and the admin server binds it again every amlAdminRetry in the +// background, while the public listener serves; READY=1 does not wait for it. +// +// Shutdown, on SIGINT or SIGTERM or when a component fails: the watchdog +// stops, then systemd is told STOPPING=1; the public listener closes and the +// consolidation scheduler drains (no job starts, no Add waits for it). The +// requests in flight and the running jobs have amlShutdownGrace to finish. +// Then the requests still running are canceled (they answer 503, which the +// platform retries) and the jobs still running are cut (their saved batches +// stay, and they resume after a restart); the canceled requests have +// amlCancelGrace more to end before their connections are closed. Then the +// admin server, kept open so the drain can be watched, closes; every +// component is waited for; and the server closes its stores and models +// within what is left of both windows (ruling R90): closing the models waits +// for their requests in flight, abandoned ones included, until the windows +// end, then cuts those still running, each of which settles its budget +// reservation at its input estimate (or releases it, unsent) within +// amlSettleGrace more. Should even that not finish, the process exits without +// them, their reservations held, and the next start settles them (the lock +// is then given back while the close goes on, as the process exits). A +// shutdown that had to cancel or cut anything fails the command (exit status +// 1) and says what. + +const ( + // amlLockName is the data root's single-instance lock file. + amlLockName = ".aml-serve.lock" + // amlMinWatchdog is the shortest watchdog interval systemd may give: the + // liveness probe needs the heartbeat, which ticks every second, to have + // ticked within the interval, and pings every half interval. + amlMinWatchdog = 2 * time.Second +) + +var ( + // amlShutdownGrace is how long the requests in flight and the running + // consolidation jobs have to finish once the shutdown starts; + // amlCancelGrace, how long the requests canceled then have to end. Tests + // shorten them. + amlShutdownGrace = 30 * time.Second + amlCancelGrace = 5 * time.Second + // amlSettleGrace is how long the model requests cut at the end of both + // windows have to settle their reservations before the process exits. + // 30 s + 5 s + 5 s stays within the unit's TimeoutStopSec=60. + amlSettleGrace = 5 * time.Second + // amlAdminRetry is how often an admin listener that could not bind is + // bound again. Tests shorten it. + amlAdminRetry = 5 * time.Second + // amlWatchdogTicker makes the watchdog's ticker, of period d: its ticks + // and its stop. Tests tick it themselves. + amlWatchdogTicker = func(d time.Duration) (<-chan time.Time, func()) { + ticker := time.NewTicker(d) + return ticker.C, ticker.Stop + } + // sdNotify tells systemd the service's state (sd_notify): nothing without + // NOTIFY_SOCKET, which stays set for the notifications to come. It sends + // one datagram and returns: the components call it on their goroutines + // (the watchdog in its loop) and it must not block them. Tests record the + // states instead. + sdNotify = func(state string) error { + _, err := daemon.SdNotify(false, state) + return err + } +) + +// lockDataRoot creates the data root if needed and takes its single-instance +// lock, an exclusive lock on /.aml-serve.lock: a file of its own, +// never a database file, whose SQLite locks are dropped when any descriptor +// on it closes. The kernel releases the lock when the process ends, however +// it ends; while another process holds it the command refuses to run. +func lockDataRoot(root string) (*flock.Flock, error) { + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, err + } + lock := flock.New(filepath.Join(root, amlLockName)) + locked, err := lock.TryLock() + if err != nil { + return nil, fmt.Errorf("lock %s: %w", lock.Path(), err) + } + if !locked { + return nil, fmt.Errorf("another aml-serve holds %s", lock.Path()) + } + return lock, nil +} + +// amlAdminServer is the HTTP server of the admin listener (aml.Server's +// AdminHandler, served on loopback, or on a private address with an admin +// token: checkAdminListen). Its write timeout outlasts the profiles it serves: +// net/http/pprof refuses a profile as long as the write timeout, and one of up +// to 4 minutes finishes within it. +func amlAdminServer(handler http.Handler, log *slog.Logger) *http.Server { + return &http.Server{ + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, + WriteTimeout: 5 * time.Minute, + IdleTimeout: 2 * time.Minute, + ErrorLog: slog.NewLogLogger(log.Handler(), slog.LevelError), + } +} + +// amlProcess is one aml-serve process: the server and its listeners. +type amlProcess struct { + server *aml.Server + log *slog.Logger + public *http.Server + // admin is nil without --admin-listen, whose address is adminAddr; its + // listener is nil while that address has not bound (bindAdmin). + admin *http.Server + adminAddr string + publicListener, adminListener net.Listener + // watchdog is systemd's watchdog interval, 0 without one; watchdogDone + // closes once the watchdog has returned (nil while it has not started). + watchdog time.Duration + watchdogDone chan struct{} +} + +// newAMLProcess binds the process's listeners for server. A public listener +// that cannot bind fails it; an admin listener that cannot bind is logged, +// and bound again later (bindAdmin). +func newAMLProcess(server *aml.Server, options amlServeOptions, log *slog.Logger, watchdog time.Duration) (*amlProcess, error) { + p := &amlProcess{server: server, log: log, public: server.HTTPServer(options.listen), watchdog: watchdog} + var err error + if p.publicListener, err = net.Listen("tcp", options.listen); err != nil { + return nil, err + } + p.logf(slog.LevelInfo, "aml-serve listening on %s", p.publicListener.Addr()) + if options.adminListen != "" { + p.admin, p.adminAddr = amlAdminServer(server.AdminHandler(), log), options.adminListen + if p.adminListener, err = net.Listen("tcp", options.adminListen); err != nil { + p.logf(slog.LevelWarn, "aml-serve: --admin-listen: %v; serving without the admin listener and binding it again every %s", err, amlAdminRetry) + } else { + p.logf(slog.LevelInfo, "aml-serve admin listening on %s", p.adminListener.Addr()) + } + } + return p, nil +} + +func (p *amlProcess) logf(level slog.Level, format string, args ...any) { + p.log.Log(context.Background(), level, fmt.Sprintf(format, args...)) +} + +// notify tells systemd state (sdNotify, which does not block); a failure is +// logged. +func (p *amlProcess) notify(state string) { + if err := sdNotify(state); err != nil { + p.logf(slog.LevelWarn, "aml-serve: telling systemd %s failed: %v", state, err) + } +} + +// serve runs the process until ctx ends (SIGINT or SIGTERM) or a component +// fails, then shuts it down. Once the shutdown starts, stopSignals gives the +// signals their default action back, so a second one ends the process at +// once. It returns the component's failure and what the shutdown had to cut. +func (p *amlProcess) serve(ctx context.Context, stopSignals func()) error { + // Serve closes a listener it served; these close the others. + defer p.publicListener.Close() + if p.adminListener != nil { + defer p.adminListener.Close() + } + sup := newSupervisor() + work, stopWork := context.WithCancel(context.Background()) + defer stopWork() + watch, stopWatch := context.WithCancel(context.Background()) + defer stopWatch() + // The admin server's retries of its bind end as it closes. + admin, stopAdmin := context.WithCancel(context.Background()) + defer stopAdmin() + if p.start(sup, work, watch, admin) == nil { + p.notify(daemon.SdNotifyReady) + select { + case <-ctx.Done(): + p.logf(slog.LevelInfo, "aml-serve: shutting down") + case <-sup.failed.Done(): + p.logf(slog.LevelError, "aml-serve: %v; shutting down", sup.err()) + } + } + stopSignals() + cut := p.shutdown(sup, stopWork, stopWatch, stopAdmin) + return errors.Join(sup.err(), cut) +} + +// start starts the components in order, each once the one before is ready: +// the background work (Run: its users queued and its heartbeat ticking), the +// admin server, the public server and systemd's watchdog. Run ends with work, +// the watchdog with watch, and an admin server still binding with admin. It +// returns the error of a component that failed to start. +func (p *amlProcess) start(sup *supervisor, work, watch, admin context.Context) error { + err := sup.start("background work", func(ready func()) error { + if err := p.server.Run(work, ready); err != nil { + return err + } + if work.Err() == nil { + return errors.New("it stopped before the shutdown") + } + return nil + }) + switch { + case err != nil || p.admin == nil: + case p.adminListener != nil: + err = sup.start("admin server", serveHTTP(p.admin, p.adminListener)) + default: + err = sup.start("admin server", p.bindAdmin(admin)) + } + if err == nil { + err = sup.start("public server", serveHTTP(p.public, p.publicListener)) + } + if err == nil && p.watchdog > 0 { + done := make(chan struct{}) + p.watchdogDone = done + err = sup.start("watchdog", func(ready func()) error { + defer close(done) + ready() + live := func(ctx context.Context) error { return amlLive(ctx, p.server, p.watchdog) } + amlWatchdog(watch, p.watchdog, live, p.notify, p.log) + return nil + }) + } + return err +} + +// serveHTTP is the component that serves server's requests from listener +// until Shutdown or Close. +func serveHTTP(server *http.Server, listener net.Listener) func(ready func()) error { + return func(ready func()) error { + ready() + if err := server.Serve(listener); !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + } +} + +// bindAdmin is the admin server's component while its address has not +// bound (R91): it is ready at once, so that the public server starts and +// the data plane serves without it, and it binds the address again every +// amlAdminRetry, quietly, until it can, then serves it as serveHTTP does +// and says so; or until ctx ends, as the admin server closes. A listener +// bound as the server closes is served and closed at once. +func (p *amlProcess) bindAdmin(ctx context.Context) func(ready func()) error { + return func(ready func()) error { + ready() + retry := time.NewTicker(amlAdminRetry) + defer retry.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-retry.C: + } + listener, err := net.Listen("tcp", p.adminAddr) + if err != nil { + continue + } + p.logf(slog.LevelInfo, "aml-serve admin listening on %s", listener.Addr()) + return serveHTTP(p.admin, listener)(func() {}) + } + } +} + +// shutdown stops the process in the phases the comment at the top of this +// file describes and returns what it had to cut, nil when nothing. +func (p *amlProcess) shutdown(sup *supervisor, stopWork, stopWatch, stopAdmin context.CancelFunc) error { + graceEnd := time.Now().Add(amlShutdownGrace) + end := graceEnd.Add(amlCancelGrace) + // The watchdog stops first, and has stopped, before systemd is told + // STOPPING=1, so that no ping follows it (the final review's parked + // T9-a). Its probe ends with it; that is quick (it calls no model). + stopWatch() + if p.watchdogDone != nil { + <-p.watchdogDone + } + p.notify(daemon.SdNotifyStopping) + p.server.Drain() + grace, endGrace := context.WithDeadline(context.Background(), graceEnd) + defer endGrace() + var cut []error + requestsLeft := false + if err := p.public.Shutdown(grace); errors.Is(err, context.DeadlineExceeded) { + requestsLeft = true + } else if err != nil { + p.logf(slog.LevelWarn, "aml-serve: closing the public listener: %v", err) + } + if err := p.server.WaitDrained(grace); err != nil { + p.logf(slog.LevelWarn, "aml-serve: the %s grace has ended with %v; cutting them: their saved batches stay, and they resume after a restart", amlShutdownGrace, err) + cut = append(cut, fmt.Errorf("cut at the end of the %s grace: %w", amlShutdownGrace, err)) + } + // Run's end cuts the jobs still running, and stops the status line and + // the heartbeat. + stopWork() + if requestsLeft { + // How many it cancels (the final review's parked T9-e). + running := requestCount(p.server.RequestsInFlight()) + p.logf(slog.LevelWarn, "aml-serve: the %s grace has ended with %s still running; canceling them", amlShutdownGrace, running) + p.server.CancelRequests() + final, endFinal := context.WithDeadline(context.Background(), end) + if p.public.Shutdown(final) != nil { + p.logf(slog.LevelError, "aml-serve: requests still running %s after they were canceled; closing their connections", amlCancelGrace) + p.public.Close() + } + endFinal() + cut = append(cut, fmt.Errorf("canceled at the end of the %s grace: %s still running", amlShutdownGrace, running)) + } + if p.admin != nil { + stopAdmin() + p.admin.Close() + } + if stuck := sup.wait(end); len(stuck) > 0 { + p.logf(slog.LevelError, "aml-serve: %s did not stop by the end of the shutdown", strings.Join(stuck, ", ")) + cut = append(cut, fmt.Errorf("did not stop: %s", strings.Join(stuck, ", "))) + } + // The models' close cuts at the windows' end what still runs (R90). + closing, endClosing := context.WithDeadline(context.Background(), end) + defer endClosing() + var closeErr error + switch { + case !closeBy(func() error { closeErr = p.server.CloseWithin(closing); return closeErr }, end.Add(amlSettleGrace)): + p.logf(slog.LevelError, "aml-serve: closing the stores and models did not finish %s after the end of the shutdown; exiting while model requests settle: their budget reservations stay held until the next start settles them", amlSettleGrace) + cut = append(cut, fmt.Errorf("closing the stores and models did not finish %s after the end of the shutdown: the model requests still settling keep their budget reservations held until the next start settles them", amlSettleGrace)) + case closeErr != nil: + p.logf(slog.LevelWarn, "aml-serve: %v", closeErr) + cut = append(cut, closeErr) + } + return errors.Join(cut...) +} + +// requestCount is n requests, in words: "1 request", "3 requests". +func requestCount(n int64) string { + if n == 1 { + return "1 request" + } + return fmt.Sprintf("%d requests", n) +} + +// closeBy runs closer and reports whether it returned by the deadline. Past +// it closer goes on alone, until it returns or the process exits; what it +// writes is the caller's to read only once closeBy has reported true. +func closeBy(closer func() error, deadline time.Time) bool { + closed := make(chan struct{}) + go func() { + closer() + close(closed) + }() + timer := time.NewTimer(time.Until(deadline)) + defer timer.Stop() + select { + case <-closed: + return true + case <-timer.C: + return false + } +} + +// supervisor owns the process's long-lived components (the Go engineering +// standard, §4.1 and §6): it starts them one at a time, each once the one +// before is ready; the first to fail ends failed, which starts the +// shutdown; and wait waits for them all. A component is never restarted +// alone: the process exits and systemd restarts it. +type supervisor struct { + group *errgroup.Group + failed context.Context + + mu sync.Mutex + running []string // the components that have not returned, in start order + first error // the first component's failure +} + +func newSupervisor() *supervisor { + group, failed := errgroup.WithContext(context.Background()) + return &supervisor{group: group, failed: failed} +} + +// start runs the component named name, which calls ready once it serves, +// and returns once it is ready, or with its failure once it has returned +// without being ready. A component that returns before it is ready fails. +// ready never blocks, and a second call does nothing. +func (s *supervisor) start(name string, run func(ready func()) error) error { + ready, exited := make(chan struct{}), make(chan error, 1) + var once sync.Once + s.mu.Lock() + s.running = append(s.running, name) + s.mu.Unlock() + s.group.Go(func() error { + err := run(func() { once.Do(func() { close(ready) }) }) + select { + case <-ready: + default: + if err == nil { + err = errors.New("it stopped before it was ready") + } + } + if err != nil { + err = fmt.Errorf("%s: %w", name, err) + } + s.exited(name, err) + exited <- err + return err + }) + select { + case <-ready: + return nil + case err := <-exited: + return err + } +} + +// exited records that the component name returned with err. +func (s *supervisor) exited(name string, err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.running = slices.DeleteFunc(s.running, func(running string) bool { return running == name }) + if s.first == nil { + s.first = err + } +} + +// err is the first component's failure, nil while none failed. +func (s *supervisor) err() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.first +} + +// wait waits for every component to return, until the deadline at most, and +// returns the names of those still running then. Past the deadline its +// waiting goroutine is left to the process's exit. +func (s *supervisor) wait(deadline time.Time) []string { + done := make(chan struct{}) + go func() { + s.group.Wait() + close(done) + }() + timer := time.NewTimer(time.Until(deadline)) + defer timer.Stop() + select { + case <-done: + return nil + case <-timer.C: + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.running) + } +} + +// amlWatchdog pings systemd's watchdog (WATCHDOG=1) every half interval, +// each time live answers within a quarter interval, until ctx ends. A failed +// probe skips its ping and is logged: systemd restarts a service whose pings +// stop for an interval. +func amlWatchdog(ctx context.Context, interval time.Duration, live func(context.Context) error, notify func(state string), log *slog.Logger) { + ticks, stop := amlWatchdogTicker(interval / 2) + defer stop() + for { + select { + case <-ctx.Done(): + return + case <-ticks: + } + probe, cancel := context.WithTimeout(ctx, interval/4) + err := live(probe) + cancel() + switch { + case ctx.Err() != nil: + return + case err != nil: + log.Warn(fmt.Sprintf("aml-serve watchdog: no ping: %v", err)) + default: + notify(daemon.SdNotifyWatchdog) + } + } +} + +// amlLive is the watchdog's liveness probe of server: its public /health +// handler, served in process, answers 200; its background work's heartbeat +// (aml.Server.Heartbeat: Run runs and the scheduler's and the store +// registry's locks are free) ticked within interval; and its budget file +// answers. It opens no store and calls no model, so a slow or failing +// provider never makes the service look dead: /ready reports those. +func amlLive(ctx context.Context, server *aml.Server, interval time.Duration) error { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "/health", nil) + if err != nil { + return err + } + health := &probeResponse{header: http.Header{}} + server.Handler().ServeHTTP(health, request) + if health.status != http.StatusOK { + return fmt.Errorf("/health answered %d", health.status) + } + beat := server.Heartbeat() + if beat.IsZero() { + return errors.New("the background work has not ticked") + } + if age := time.Since(beat); age > interval { + return fmt.Errorf("the background work last ticked %s ago", age.Round(time.Millisecond)) + } + if _, err := server.Accounts(ctx); err != nil { + return fmt.Errorf("the budget file does not answer: %w", err) + } + return nil +} + +// probeResponse keeps the status a handler answers the liveness probe with; +// the body is dropped. +type probeResponse struct { + header http.Header + status int +} + +func (r *probeResponse) Header() http.Header { return r.header } + +func (r *probeResponse) WriteHeader(status int) { + if r.status == 0 { + r.status = status + } +} + +func (r *probeResponse) Write(body []byte) (int, error) { + r.WriteHeader(http.StatusOK) + return len(body), nil +} diff --git a/cmd/memory/aml_test.go b/cmd/memory/aml_test.go new file mode 100644 index 000000000..c401928e1 --- /dev/null +++ b/cmd/memory/aml_test.go @@ -0,0 +1,1935 @@ +package memory + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "reflect" + "regexp" + "runtime" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gofrs/flock" + "github.com/mnemon-dev/mnemon/internal/aml" + "github.com/mnemon-dev/mnemon/internal/memoryagent" +) + +func TestAMLServeIsAMemoryAgentSubcommand(t *testing.T) { + command, _, err := rootCmd.Find([]string{"memory-agent", "aml-serve"}) + if err != nil || command.Name() != "aml-serve" || command.Parent().Name() != "memory-agent" { + t.Fatal(command, err) + } + for flag, value := range map[string]string{ + "listen": "127.0.0.1:8090", "data-root": "", "memory-config": "", "auth-key-env": "AML_API_KEY", "consolidate-on-add": "false", + "max-open-stores": "64", "search-timeout": "10m0s", "consolidate": "off", "consolidation-workers": "4", "admin-listen": "", + "admin-token-env": "", "max-concurrent-adds": "0", "max-concurrent-searches": "0", "log-format": "text", + } { + if f := command.Flags().Lookup(flag); f == nil || f.DefValue != value { + t.Fatal(flag, f) + } + } +} + +// amlFlags parses args, after an absolute --data-root, as aml-serve's flags. +func amlFlags(t *testing.T, args ...string) (amlServeOptions, error) { + t.Helper() + command := newAMLServeCommand() + if err := command.ParseFlags(append([]string{"--data-root", t.TempDir()}, args...)); err != nil { + t.Fatal(err) + } + return amlServeFlags(command) +} + +// executeAMLServe runs aml-serve with args to its end. +func executeAMLServe(args ...string) error { + command := newAMLServeCommand() + command.SetArgs(args) + command.SilenceUsage, command.SilenceErrors = true, true + command.SetOut(io.Discard) + command.SetErr(io.Discard) + return command.Execute() +} + +func TestConsolidateFlagValues(t *testing.T) { + for args, want := range map[string]aml.ConsolidationMode{ + "": aml.ConsolidationOff, + "--consolidate off": aml.ConsolidationOff, + "--consolidate on-add": aml.ConsolidationOnAdd, + "--consolidate background": aml.ConsolidationBackground, + "--consolidate-on-add": aml.ConsolidationOnAdd, // the older spelling + "--consolidate-on-add=true": aml.ConsolidationOnAdd, + "--consolidate-on-add=false": aml.ConsolidationOff, + } { + options, err := amlFlags(t, strings.Fields(args)...) + if err != nil || options.consolidation != want { + t.Fatalf("%q: %q %v", args, options.consolidation, err) + } + } + for args, want := range map[string]string{ + "--consolidate bogus": `--consolidate must be off, on-add or background, not "bogus"`, + "--consolidate=": `--consolidate must be off, on-add or background, not ""`, + "--consolidate-on-add --consolidate background": "--consolidate-on-add is the older spelling of --consolidate on-add: give one of them", + "--consolidate on-add --consolidate-on-add": "--consolidate-on-add is the older spelling of --consolidate on-add: give one of them", + "--consolidate-on-add=false --consolidate off": "--consolidate-on-add is the older spelling of --consolidate on-add: give one of them", + "--consolidation-workers 0": "--consolidation-workers must be at least 1", + "--max-concurrent-adds -1": "--max-concurrent-adds must not be negative", + "--max-concurrent-searches -1": "--max-concurrent-searches must not be negative", + "--log-format yaml": `--log-format must be text or json, not "yaml"`, + } { + if _, err := amlFlags(t, strings.Fields(args)...); err == nil || err.Error() != want { + t.Fatalf("%q: %v", args, err) + } + } + // The command refuses them before it takes the data root's lock. + dir := t.TempDir() + t.Setenv("AML_TEST_KEY", "k") + if err := executeAMLServe("--data-root", dir, "--memory-config", filepath.Join(dir, "memory.json"), "--auth-key-env", "AML_TEST_KEY", "--consolidate", "bogus"); err == nil || !strings.Contains(err.Error(), "--consolidate must be") { + t.Fatal(err) + } + if entries, err := os.ReadDir(dir); err != nil || len(entries) != 0 { + t.Fatal("the refused command touched the data root", entries, err) + } +} + +// --provider-keys-file names the account's key file (ruling R98), which +// aml-serve reads at its start and writes for the admin listener, and +// aml-inspect reads: an absolute path, or none. +func TestProviderKeysFileFlag(t *testing.T) { + file := filepath.Join(t.TempDir(), "provider-keys.json") + if options, err := amlFlags(t, "--provider-keys-file", file); err != nil || options.providerKeysFile != file { + t.Fatal(options.providerKeysFile, err) + } + if options, err := amlFlags(t); err != nil || options.providerKeysFile != "" { + t.Fatal(options.providerKeysFile, err) + } + if _, err := amlFlags(t, "--provider-keys-file", "provider-keys.json"); err == nil || err.Error() != "--provider-keys-file must be an absolute path" { + t.Fatal(err) + } + if options, err := inspectFlags(t, "--provider-keys-file", file); err != nil || options.providerKeysFile != file { + t.Fatal(options.providerKeysFile, err) + } + if _, err := inspectFlags(t, "--provider-keys-file", "provider-keys.json"); err == nil || err.Error() != "--provider-keys-file must be an absolute path" { + t.Fatal(err) + } + // A key file the start refuses stops aml-serve, naming the file and + // nothing it holds. + dir := t.TempDir() + t.Setenv("AML_TEST_KEY", "k") + t.Setenv("DEEPSEEK_API_KEY", "test-deepseek") + t.Setenv("TYPESAFE_API_KEY", "test-typesafe") + config, err := readMemoryConfig(deployFile(t, "memory-config.industrial.json")) + if err != nil { + t.Fatal(err) + } + config.BudgetFile = filepath.Join(dir, "model-budget.db") + raw, _ := json.Marshal(config) + configPath := filepath.Join(dir, "memory-config.json") + if err := os.WriteFile(configPath, raw, 0o600); err != nil { + t.Fatal(err) + } + const planted = "sk-planted-in-the-file-0001" + if err := os.WriteFile(file, []byte(`{"version":1,"keys":{"deepseek":"`+planted+`"},"extra":true}`), 0o600); err != nil { + t.Fatal(err) + } + err = executeAMLServe("--data-root", filepath.Join(dir, "data"), "--memory-config", configPath, "--auth-key-env", "AML_TEST_KEY", "--provider-keys-file", file) + if err == nil || !strings.Contains(err.Error(), file) || strings.Contains(err.Error(), planted) { + t.Fatal(err) + } +} + +// --settings-file names the file of the caps that change at run time +// (ruling R99): aml-serve's budgets, and aml-inspect's own debug caps, which +// need its debug budget. An absolute path, or none. +func TestSettingsFileFlag(t *testing.T) { + file := filepath.Join(t.TempDir(), "settings.json") + if options, err := amlFlags(t, "--settings-file", file); err != nil || options.settingsFile != file { + t.Fatal(options.settingsFile, err) + } + if options, err := amlFlags(t); err != nil || options.settingsFile != "" { + t.Fatal(options.settingsFile, err) + } + if _, err := amlFlags(t, "--settings-file", "settings.json"); err == nil || err.Error() != "--settings-file must be an absolute path" { + t.Fatal(err) + } + debug := filepath.Join(t.TempDir(), "debug-budget.db") + if options, err := inspectFlags(t, "--debug-budget-file", debug, "--settings-file", file); err != nil || options.settingsFile != file { + t.Fatal(options.settingsFile, err) + } + if _, err := inspectFlags(t, "--debug-budget-file", debug, "--settings-file", "settings.json"); err == nil || err.Error() != "--settings-file must be an absolute path" { + t.Fatal(err) + } + if _, err := inspectFlags(t, "--settings-file", file); err == nil || err.Error() != "--settings-file needs --debug-budget-file" { + t.Fatal(err) + } + // A settings file the start refuses stops aml-serve, naming the file. + dir := t.TempDir() + t.Setenv("AML_TEST_KEY", "k") + t.Setenv("DEEPSEEK_API_KEY", "test-deepseek") + t.Setenv("TYPESAFE_API_KEY", "test-typesafe") + config, err := readMemoryConfig(deployFile(t, "memory-config.industrial.json")) + if err != nil { + t.Fatal(err) + } + config.BudgetFile = filepath.Join(dir, "model-budget.db") + raw, _ := json.Marshal(config) + configPath := filepath.Join(dir, "memory-config.json") + if err := os.WriteFile(configPath, raw, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte(`{"version":1,"budgets":{"deepseek":0}}`), 0o600); err != nil { + t.Fatal(err) + } + err = executeAMLServe("--data-root", filepath.Join(dir, "data"), "--memory-config", configPath, "--auth-key-env", "AML_TEST_KEY", "--settings-file", file) + if err == nil || !strings.Contains(err.Error(), file) { + t.Fatal(err) + } +} + +// The E1 cases of a listener's address, which aml-serve's --admin-listen +// (TestAdminListenRules) and aml-inspect's --listen (TestInspectAuthAndListen) +// both follow: loopback addresses need no token, private ones need it, and +// every other is refused, token or not. +var ( + loopbackListens = []string{"127.0.0.1:8091", "127.0.0.1:0", "127.0.0.2:8091", "[::1]:8091", "[::ffff:127.0.0.1]:8091"} + privateListens = []string{"10.0.0.5:8091", "172.16.0.1:8091", "172.31.255.255:8091", "192.168.1.2:8091", "[fd00::1]:8091", "[::ffff:10.0.0.5]:8091"} + // Unspecified, link-local, shared (100.64/10), public and documentation + // addresses, the neighbours of the private ranges, host names, and what + // is not an address and a port. + refusedListens = []string{ + "0.0.0.0:8091", "[::]:8091", ":8091", "[::ffff:0.0.0.0]:8091", + "169.254.1.1:8091", "[fe80::1]:8091", "[fe80::1%eth0]:8091", "[fd00::1%eth0]:8091", + "100.64.0.1:8091", "203.0.113.5:8091", "[::ffff:203.0.113.5]:8091", "[2001:db8::1]:8091", "172.32.0.1:8091", "172.15.255.255:8091", "11.0.0.1:8091", "192.169.0.1:8091", + "localhost:8091", "example.com:8091", "10.0.0.5", "10.0.0.5:", "10.0.0.5:99999", "10.0.0.5:http", "[10.0.0.5]", "http://10.0.0.5:8091", + } +) + +// The admin endpoints are served on a loopback address, where an admin token +// is optional, or on a private one (10/8, 172.16/12, 192.168/16, fc00::/7), +// where it is required. Every other address is refused, with a token or +// without, as is a host name or an address with no port. A flag that names a +// variable needs it to hold 32 bytes at least. +func TestAdminListenRules(t *testing.T) { + const variable = "AML_TEST_ADMIN_TOKEN" + token := strings.Repeat("t", 32) + wantError := func(t *testing.T, err error, parts ...string) { + t.Helper() + if err == nil { + t.Fatal("no error") + } + for _, part := range parts { + if !strings.Contains(err.Error(), part) { + t.Fatalf("%q lacks %q", err, part) + } + } + } + + // Loopback addresses, and none at all, need no token. An IPv4 address + // spelled as an IPv6 one is the IPv4 address. + for _, addr := range append([]string{""}, loopbackListens...) { + if options, err := amlFlags(t, "--admin-listen", addr); err != nil || options.adminListen != addr || options.adminToken != "" { + t.Fatal(addr, options.adminListen, err) + } + // A token is optional there, not forbidden. + t.Setenv(variable, token) + if options, err := amlFlags(t, "--admin-listen", addr, "--admin-token-env", variable); err != nil || options.adminToken != token { + t.Fatal(addr, err) + } + } + + // Private addresses are refused without a token, and accepted with one. + for _, addr := range privateListens { + _, err := amlFlags(t, "--admin-listen", addr) + wantError(t, err, "--admin-listen", "private", "--admin-token-env") + t.Setenv(variable, token) + if options, err := amlFlags(t, "--admin-listen", addr, "--admin-token-env", variable); err != nil || options.adminListen != addr || options.adminToken != token { + t.Fatal(addr, options.adminListen, err) + } + } + + // Everything else is refused, whatever the token. + for _, addr := range refusedListens { + _, err := amlFlags(t, "--admin-listen", addr) + wantError(t, err, "--admin-listen must be") + t.Setenv(variable, token) + _, err = amlFlags(t, "--admin-listen", addr, "--admin-token-env", variable) + wantError(t, err, "--admin-listen must be") + } + + // The token: the variable the flag names must hold 32 bytes at + // least, on any address; what the error says never repeats a value. + const missing = "AML_TEST_NO_SUCH_ADMIN_TOKEN" + for _, addr := range []string{"", "127.0.0.1:8091", "10.0.0.5:8091"} { + t.Setenv(variable, "") + _, err := amlFlags(t, "--admin-listen", addr, "--admin-token-env", variable) + wantError(t, err, "--admin-token-env", variable, "unset or empty") + os.Unsetenv(missing) + _, err = amlFlags(t, "--admin-listen", addr, "--admin-token-env", missing) + wantError(t, err, "--admin-token-env", missing, "unset or empty") + short := strings.Repeat("s", 31) + t.Setenv(variable, short) + _, err = amlFlags(t, "--admin-listen", addr, "--admin-token-env", variable) + wantError(t, err, "--admin-token-env", variable, "32 bytes") + if strings.Contains(err.Error(), short) { + t.Fatal("the error shows the token:", err) + } + } + // Even with no admin listener, a flag that names a variable is checked: an + // operator who set it means to have a token. + t.Setenv(variable, "") + _, err := amlFlags(t, "--admin-token-env", variable) + wantError(t, err, "--admin-token-env", variable) + + // The command refuses them before it takes the data root's lock. + dir := t.TempDir() + t.Setenv("AML_TEST_KEY", "k") + err = executeAMLServe("--data-root", dir, "--memory-config", filepath.Join(dir, "memory.json"), "--auth-key-env", "AML_TEST_KEY", "--admin-listen", "10.0.0.5:8091") + wantError(t, err, "--admin-listen", "private") + err = executeAMLServe("--data-root", dir, "--memory-config", filepath.Join(dir, "memory.json"), "--auth-key-env", "AML_TEST_KEY", "--admin-listen", "127.0.0.1:0", "--admin-token-env", missing) + wantError(t, err, "--admin-token-env", missing) + if entries, err := os.ReadDir(dir); err != nil || len(entries) != 0 { + t.Fatal("a refused command touched the data root", entries, err) + } +} + +// Parked T14-d (final review): the admin token and the API key are two +// secrets: --admin-token-env naming the variable --auth-key-env names would +// make the API key, which every client holds, the admin token. It is +// refused, whatever the variable holds, and the error shows no value. +func TestAdminTokenIsNotTheAPIKey(t *testing.T) { + const shared = "AML_TEST_SHARED_SECRET" + secret := strings.Repeat("k", 40) + t.Setenv(shared, secret) + _, err := amlFlags(t, "--admin-listen", "127.0.0.1:8091", "--admin-token-env", shared, "--auth-key-env", shared) + if err == nil || !strings.Contains(err.Error(), "--admin-token-env") || !strings.Contains(err.Error(), "--auth-key-env") || strings.Contains(err.Error(), secret) { + t.Fatal(err) + } + // Two variables, even holding one value, are two flags' business: the + // kit writes two secrets. + t.Setenv("AML_TEST_OTHER_SECRET", secret) + if _, err := amlFlags(t, "--admin-listen", "127.0.0.1:8091", "--admin-token-env", "AML_TEST_OTHER_SECRET", "--auth-key-env", shared); err != nil { + t.Fatal(err) + } +} + +// One aml-serve per data root: a second one refuses to start while the first +// holds /.aml-serve.lock, before it opens the budget file or a +// store; a command that fails after taking the lock gives it back. +func TestAMLServeRefusesSecondInstance(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, "memory.json") + if err := os.WriteFile(config, []byte(`{"version":1,"strategy":"simple"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("AML_TEST_KEY", "k") + args := []string{"--data-root", dir, "--memory-config", config, "--auth-key-env", "AML_TEST_KEY"} + path := filepath.Join(dir, ".aml-serve.lock") + held := flock.New(path) + if locked, err := held.TryLock(); !locked || err != nil { + t.Fatal(locked, err) + } + if err := executeAMLServe(args...); err == nil || err.Error() != "another aml-serve holds "+path { + t.Fatal(err) + } + if err := held.Unlock(); err != nil { + t.Fatal(err) + } + // Free, the lock is taken; the server then refuses the configuration, + // which names no models, and the lock is given back. + if err := executeAMLServe(args...); err == nil || !strings.Contains(err.Error(), "requires planner and decision models") { + t.Fatal(err) + } + if locked, err := held.TryLock(); !locked || err != nil { + t.Fatal("the failed command kept the lock", locked, err) + } + held.Unlock() +} + +func TestAMLServeRefusesBeforeListening(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, "memory.json") + if err := os.WriteFile(config, []byte(`{"version":1,"strategy":"simple"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("AML_TEST_KEY", "") + t.Setenv("AML_TEST_SET", "k") + for _, c := range []struct { + args []string + want string + }{ + {[]string{"--data-root", "relative", "--memory-config", config, "--auth-key-env", "AML_TEST_SET"}, "absolute --data-root"}, + {[]string{"--data-root", dir, "--memory-config", config, "--auth-key-env", "AML_TEST_KEY"}, "nonempty API key in $AML_TEST_KEY"}, + {[]string{"--data-root", dir, "--memory-config", "memory.json", "--auth-key-env", "AML_TEST_SET"}, "absolute file path"}, + // The configuration is checked by making an agent: no models. + {[]string{"--data-root", dir, "--memory-config", config, "--auth-key-env", "AML_TEST_SET"}, "requires planner and decision models"}, + {[]string{"--data-root", dir, "--memory-config", config, "--auth-key-env", "AML_TEST_SET", "--max-open-stores", "0"}, "--max-open-stores"}, + {[]string{"--data-root", dir, "--memory-config", config, "--auth-key-env", "AML_TEST_SET", "--search-timeout", "59s"}, "--search-timeout must be between 1m and 18m"}, + {[]string{"--data-root", dir, "--memory-config", config, "--auth-key-env", "AML_TEST_SET", "--search-timeout", "19m"}, "--search-timeout must be between 1m and 18m"}, + } { + command := newAMLServeCommand() + command.SetArgs(c.args) + command.SilenceUsage, command.SilenceErrors = true, true + if err := command.Execute(); err == nil || !strings.Contains(err.Error(), c.want) { + t.Fatal(c.args, err) + } + } +} + +// A watchdog interval the liveness probe cannot keep up with, or one systemd +// could not have set, refuses to start before the data root is locked. +func TestAMLServeRefusesAWatchdogItCannotPing(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, "memory.json") + if err := os.WriteFile(config, []byte(`{"version":1,"strategy":"simple"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("AML_TEST_KEY", "k") + for usec, want := range map[string]string{ + "1999999": "systemd's watchdog interval 1.999999s is below 2s: set WatchdogSec to at least 2s", + "soon": "systemd watchdog: error converting WATCHDOG_USEC", + } { + t.Setenv("WATCHDOG_USEC", usec) + if err := executeAMLServe("--data-root", dir, "--memory-config", config, "--auth-key-env", "AML_TEST_KEY"); err == nil || !strings.Contains(err.Error(), want) { + t.Fatal(usec, err) + } + } + if _, err := os.Stat(filepath.Join(dir, ".aml-serve.lock")); !os.IsNotExist(err) { + t.Fatal("the refused command took the lock", err) + } +} + +// deployFile is the absolute path of a file of the deployment kit. +func deployFile(t *testing.T, name string) string { + t.Helper() + path, err := filepath.Abs(filepath.Join("..", "..", "deploy", "aml", name)) + if err != nil { + t.Fatal(err) + } + return path +} + +// The industrial configuration is what the runbook tells the operator to +// install: strict decoding must accept it, it must say what the submission +// says it does, and a server must start on it (no network is used). +func TestIndustrialConfigLoadsAndStarts(t *testing.T) { + config, err := readMemoryConfig(deployFile(t, "memory-config.industrial.json")) + if err != nil { + t.Fatal(err) + } + want := memoryagent.Config{ + Version: 1, + Strategy: "simple", + // The judged View: no fusion with the plain search ranking, so the + // View is the records JEV judged (maxItems and maxCharacters only + // cap it), delivered in time order after the consolidation blocks. + View: &memoryagent.ViewConfig{MaxItems: 100, MaxCharacters: 240000, Order: "time"}, + // Layered consolidation: DeepSeek extracts each batch alone, reading + // every record whole whatever its role (even reading), and JEV links + // it into the memory. No index routes: Index stays unset. + Consolidation: &memoryagent.ConsolidationConfig{Mode: "layered", EvenReading: true}, + // DeepSeek without thinking: a thinking planner can return empty plans + // under the 200/400-token planning caps. + Planner: &memoryagent.ModelConfig{Kind: "deepseek", Model: "deepseek-flash", KeyEnv: "DEEPSEEK_API_KEY"}, + Judge: &memoryagent.ModelConfig{Kind: "jev", Model: "jev-1.13.0", KeyEnv: "TYPESAFE_API_KEY"}, + // The extraction of a full batch may take minutes; the default 90 s + // would cut it off. + Consolidator: &memoryagent.ModelConfig{Kind: "deepseek", Model: "deepseek-flash", KeyEnv: "DEEPSEEK_API_KEY", TimeoutSeconds: 600}, + // BM25 only, no Ollama: without an embedding section the cue searches + // match words, the consolidation layer and the linking rank by shared + // words, and an Add embeds nothing. + Embedding: nil, + BudgetFile: "/var/lib/mnemon-aml/model-budget.db", + // The evaluation caps: JEV in USD, DeepSeek in CNY. + Budgets: map[string]float64{"jev": 60, "deepseek": 600}, + } + if !reflect.DeepEqual(config, want) { + got, _ := json.MarshalIndent(config, "", " ") + wanted, _ := json.MarshalIndent(want, "", " ") + t.Fatalf("industrial config\n got %s\nwant %s", got, wanted) + } + + // The same file with a writable budget path: NewServer builds the real + // providers and probes the agents, as aml-serve does at startup. + config.BudgetFile = filepath.Join(t.TempDir(), "model-budget.db") + serve := func() error { + server, err := aml.NewServer(aml.ServerConfig{DataRoot: t.TempDir(), MemoryConfig: config, AuthKey: "k", ConsolidateOnAdd: true, MaxOpenStores: 16}) + if err == nil { + server.Close() + } + return err + } + t.Setenv("DEEPSEEK_API_KEY", "test-deepseek") + t.Setenv("TYPESAFE_API_KEY", "test-typesafe") + if err := serve(); err != nil { + t.Fatal(err) + } + // The runbook's troubleshooting entry: a missing key refuses to start. + t.Setenv("TYPESAFE_API_KEY", "") + if err := serve(); err == nil || !strings.Contains(err.Error(), "credential environment variable") { + t.Fatal(err) + } +} + +// systemdUnit is a unit file's settings: by section, each key's values in +// file order (a key may be given more than once). +type systemdUnit map[string]map[string][]string + +// readUnit parses the kit's unit file name. Comments and blank lines are +// skipped. A setting continued on the next line is refused, so that every +// setting, ExecStart above all, is one line, the same for every instance. +func readUnit(t *testing.T, name string) systemdUnit { + t.Helper() + raw, err := os.ReadFile(deployFile(t, name)) + if err != nil { + t.Fatal(err) + } + unit, section := systemdUnit{}, "" + // A Windows checkout may end the lines with \r\n. + for _, line := range strings.Split(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") { + line = strings.TrimSpace(line) + switch { + case line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";"): + case strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]"): + section = strings.Trim(line, "[]") + if unit[section] == nil { + unit[section] = map[string][]string{} + } + default: + key, value, ok := strings.Cut(line, "=") + if !ok || section == "" || strings.HasSuffix(value, `\`) { + t.Fatalf("%s: %q is not a one-line setting of a section", name, line) + } + key = strings.TrimSpace(key) + unit[section][key] = append(unit[section][key], strings.TrimSpace(value)) + } + } + return unit +} + +// get is the one value of key in section; a missing or repeated key fails. +func (u systemdUnit) get(t *testing.T, section, key string) string { + t.Helper() + if values := u[section][key]; len(values) == 1 { + return values[0] + } + t.Fatalf("[%s] %s is %q, want one value", section, key, u[section][key]) + return "" +} + +// systemdSpan reads a systemd time span of one number and a unit: none +// (seconds), ms, s or min. +func systemdSpan(t *testing.T, span string) time.Duration { + t.Helper() + match := regexp.MustCompile(`^([0-9]+)(ms|s|min)?$`).FindStringSubmatch(span) + if match == nil { + t.Fatalf("time span %q is not one number and a unit", span) + } + n, _ := time.ParseDuration(match[1] + "s") + switch match[2] { + case "ms": + return n / 1000 + case "min": + return n * 60 + } + return n +} + +// execArgs is the command line systemd runs for execStart in instance, whose +// EnvironmentFile holds env: %i is the instance and ${NAME} is NAME's value. +// The templates use neither %I, which would turn the dashes of a name into +// slashes, nor $NAME, which systemd splits at whitespace. +func execArgs(t *testing.T, execStart, instance string, env map[string]string) []string { + t.Helper() + line := strings.ReplaceAll(execStart, "%i", instance) + var unknown []string + line = regexp.MustCompile(`\$\{([A-Za-z_][A-Za-z0-9_]*)\}`).ReplaceAllStringFunc(line, func(ref string) string { + value, ok := env[ref[2:len(ref)-1]] + if !ok { + unknown = append(unknown, ref) + } + return value + }) + if len(unknown) > 0 || strings.ContainsAny(line, "$%") { + t.Fatalf("ExecStart %q uses %q, which the instance's environment file does not set, or a $ or %% this test does not expand", execStart, unknown) + } + return strings.Fields(line) +} + +// unitSandbox is the sandboxing aml-serve@.service sets and +// aml-watch@.service repeats. +var unitSandbox = []string{ + "UMask", "NoNewPrivileges", "PrivateTmp", "PrivateDevices", "ProtectSystem", "ProtectHome", + "ProtectKernelTunables", "ProtectKernelModules", "ProtectControlGroups", "RestrictSUIDSGID", + "LockPersonality", "RestrictRealtime", +} + +// The instance templates must run aml-serve and aml-watch with flags the +// commands have once systemd has filled an instance in, and carry the +// settings the runbook and the memory cloud design rely on. +func TestServiceUnitFlagsParse(t *testing.T) { + // No file of the kit mentions Ollama: retrieval is BM25 only. + err := filepath.WalkDir(deployFile(t, ""), func(file string, entry os.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + raw, err := os.ReadFile(file) + if err == nil && strings.Contains(strings.ToLower(string(raw)), "ollama") { + t.Errorf("%s mentions Ollama", file) + } + return err + }) + if err != nil { + t.Fatal(err) + } + + // A dedicated instance's two environment files, as mnemon-instance new + // writes them, each with only its own names: env, the engine's (the record, + // the addresses, the flag values, the Go memory limit and the secrets), and + // watch.env, the watcher's five. A template that reads a variable its file + // does not set fails here. + const instance = "eval" + env := map[string]string{ + "AML_LAYOUT": "dedicated", "AML_SLOT": "0", "AML_STATUS": "active", + "AML_LISTEN": "127.0.0.1:8090", "AML_ADMIN_LISTEN": "10.0.0.5:8091", + "AML_MAX_ADDS": "16", "AML_MAX_SEARCHES": "16", "AML_MAX_OPEN_STORES": "16", "AML_WORKERS": "4", + "GOMEMLIMIT": "32768MiB", "AML_API_KEY": "mnm_api-key", "MNEMON_ADMIN_TOKEN": "mnm_admin-token", + "DEEPSEEK_API_KEY": "sk-deepseek", "TYPESAFE_API_KEY": "ts-jev", + } + watchEnv := map[string]string{ + "AML_ALERT_WEBHOOK": "https://api.telegram.org/bot123:token/sendMessage?chat_id=42", "AML_ALERT_KIND": "telegram", + "AML_PUBLIC_URL": "https://eval.example.com", "AML_ADMIN_LISTEN": env["AML_ADMIN_LISTEN"], "MNEMON_ADMIN_TOKEN": env["MNEMON_ADMIN_TOKEN"], + } + + serve := readUnit(t, "aml-serve@.service") + args := execArgs(t, serve.get(t, "Service", "ExecStart"), instance, env) + if len(args) < 3 || !slices.Equal(args[:3], []string{"/usr/local/bin/mnemon", "memory-agent", "aml-serve"}) { + t.Fatal("aml-serve@.service runs", args) + } + command := newAMLServeCommand() + if err := command.ParseFlags(args[3:]); err != nil { + t.Fatal(err) + } + if rest := command.Flags().Args(); len(rest) > 0 { + t.Fatal("aml-serve@.service passes arguments that are not flags:", rest) + } + for flag, want := range map[string]string{ + "listen": "127.0.0.1:8090", + "admin-listen": "10.0.0.5:8091", + "admin-token-env": "MNEMON_ADMIN_TOKEN", + "auth-key-env": "AML_API_KEY", + "data-root": "/var/lib/mnemon-aml/eval/data", + "memory-config": "/etc/mnemon-aml/eval/memory-config.json", + "consolidate": "background", + "consolidation-workers": "4", + "max-open-stores": "16", + "max-concurrent-adds": "16", + "max-concurrent-searches": "16", + "log-format": "json", + "provider-keys-file": "/var/lib/mnemon-aml/eval/provider-keys.json", + "settings-file": "/var/lib/mnemon-aml/eval/settings.json", + } { + if f := command.Flags().Lookup(flag); !f.Changed || f.Value.String() != want { + t.Errorf("aml-serve@.service passes --%s %q, want %q", flag, f.Value.String(), want) + } + } + // The key file is in the instance's state, beside the data root and never + // in it, so that a snapshot of the data never holds a key. + if keys, data := command.Flags().Lookup("provider-keys-file").Value.String(), command.Flags().Lookup("data-root").Value.String(); path.Dir(keys) != path.Dir(data) || strings.HasPrefix(keys, data+"/") { + t.Errorf("--provider-keys-file %s is not beside --data-root %s", keys, data) + } + if settings, data := command.Flags().Lookup("settings-file").Value.String(), command.Flags().Lookup("data-root").Value.String(); path.Dir(settings) != path.Dir(data) { + t.Errorf("--settings-file %s is not beside --data-root %s", settings, data) + } + if command.Flags().Changed("consolidate-on-add") { + t.Error("aml-serve@.service passes --consolidate-on-add") + } + // The admin listener binds the host's WireGuard address: the unit starts + // wg0 with it and after it, but never needs it (R91: a Requires= would + // stop the evaluation with the overlay, and aml-serve serves its public + // listener while the admin one waits for wg0). + for _, key := range []string{"Wants", "After"} { + if values := strings.Fields(serve.get(t, "Unit", key)); !slices.Contains(values, "wg-quick@wg0.service") { + t.Errorf("aml-serve@.service %s=%q lacks wg-quick@wg0.service", key, values) + } + } + for _, key := range []string{"Requires", "Requisite", "BindsTo", "PartOf"} { + if values, ok := serve["Unit"][key]; ok { + t.Errorf("aml-serve@.service %s=%q: the data plane must not depend on the overlay", key, values) + } + } + // The admin listener binds the host's private overlay address, which + // aml-serve accepts only with the admin token. + if err := checkAdminListen(env["AML_ADMIN_LISTEN"], true); err != nil { + t.Fatal(err) + } + if checkAdminListen(env["AML_ADMIN_LISTEN"], false) == nil { + t.Fatal("a private admin listener is accepted without a token") + } + for _, s := range []struct{ section, key, want string }{ + {"Unit", "StartLimitIntervalSec", "0"}, + {"Service", "Type", "notify"}, + {"Service", "NotifyAccess", "main"}, + {"Service", "Restart", "always"}, + {"Service", "RestartSec", "5"}, + {"Service", "TimeoutStopSec", "60"}, + {"Service", "User", "mnemon"}, + {"Service", "Group", "mnemon"}, + {"Service", "EnvironmentFile", "/etc/mnemon-aml/%i/env"}, + // The data root and the budget file are under /var/lib/mnemon-aml//. + {"Service", "StateDirectory", "mnemon-aml/%i"}, + {"Service", "StateDirectoryMode", "0700"}, + {"Service", "LimitNOFILE", "65536"}, + {"Install", "WantedBy", "multi-user.target"}, + } { + if got := serve.get(t, s.section, s.key); got != s.want { + t.Errorf("aml-serve@.service [%s] %s=%s, want %s", s.section, s.key, got, s.want) + } + } + if watchdog := systemdSpan(t, serve.get(t, "Service", "WatchdogSec")); watchdog < amlMinWatchdog { + t.Errorf("WatchdogSec %s is below the %s aml-serve accepts", watchdog, amlMinWatchdog) + } + // The shutdown (aml_process.go) fits in TimeoutStopSec: the requests in + // flight and the scheduler's drain share amlShutdownGrace, the requests + // canceled then have amlCancelGrace more, and the stores and models close + // within the same window, the model requests cut at its end settling + // within amlSettleGrace (R90). + if stop, shutdown := systemdSpan(t, serve.get(t, "Service", "TimeoutStopSec")), amlShutdownGrace+amlCancelGrace+amlSettleGrace; stop <= shutdown { + t.Errorf("TimeoutStopSec %s does not outlast the shutdown's %s", stop, shutdown) + } + // The limits are per instance, in its drop-in, so the template is the + // same for every instance. + for _, key := range []string{"MemoryMax", "CPUQuota"} { + if _, ok := serve["Service"][key]; ok { + t.Errorf("aml-serve@.service sets %s, which belongs in the instance's limits.conf drop-in", key) + } + } + + watch := readUnit(t, "aml-watch@.service") + args = execArgs(t, watch.get(t, "Service", "ExecStart"), instance, watchEnv) + if len(args) < 3 || !slices.Equal(args[:3], []string{"/usr/local/bin/mnemon", "memory-agent", "aml-watch"}) { + t.Fatal("aml-watch@.service runs", args) + } + command = newAMLWatchCommand() + if err := command.ParseFlags(args[3:]); err != nil { + t.Fatal(err) + } + if rest := command.Flags().Args(); len(rest) > 0 { + t.Fatal("aml-watch@.service passes arguments that are not flags:", rest) + } + for flag, want := range map[string]string{ + "admin": "http://" + watchEnv["AML_ADMIN_LISTEN"], + "admin-token-env": "MNEMON_ADMIN_TOKEN", + "public-url": watchEnv["AML_PUBLIC_URL"], + "instance": instance, + "state": "/var/lib/mnemon-aml-watch/eval/state.json", + } { + if f := command.Flags().Lookup(flag); !f.Changed || f.Value.String() != want { + t.Errorf("aml-watch@.service passes --%s %q, want %q", flag, f.Value.String(), want) + } + } + state := command.Flags().Lookup("state").Value.String() + if !path.IsAbs(state) || path.Clean(state) != state || !strings.HasPrefix(state, "/var/lib/mnemon-aml-watch/") { + t.Errorf("--state %q is not an absolute path under /var/lib/mnemon-aml-watch/", state) + } + for flag, value := range map[string]string{"--admin": "http://" + watchEnv["AML_ADMIN_LISTEN"], "--public-url": watchEnv["AML_PUBLIC_URL"]} { + if _, err := watchBaseURL(flag, value); err != nil { + t.Error(err) + } + } + if !watchInstance.MatchString(instance) { + t.Errorf("aml-watch refuses --instance %q", instance) + } + for _, s := range []struct{ key, want string }{ + {"Type", "oneshot"}, + {"User", "mnemon"}, + {"Group", "mnemon"}, + {"EnvironmentFile", "/etc/mnemon-aml/%i/watch.env"}, + {"StateDirectory", "mnemon-aml-watch/%i"}, + {"StateDirectoryMode", "0700"}, + // A wedged run is killed, which gives its run lock back. + {"TimeoutStartSec", "90"}, + } { + if got := watch.get(t, "Service", s.key); got != s.want { + t.Errorf("aml-watch@.service %s=%s, want %s", s.key, got, s.want) + } + } + // The timer runs the watcher, which must run while aml-serve is down. + if _, ok := watch["Install"]; ok { + t.Error("aml-watch@.service has an [Install] section; its timer starts it") + } + if requires, ok := watch["Unit"]["Requires"]; ok { + t.Errorf("aml-watch@.service requires %q: the watcher must run while aml-serve is down", requires) + } + for _, key := range unitSandbox { + if want := serve["Service"][key]; len(want) == 0 || !slices.Equal(watch["Service"][key], want) { + t.Errorf("%s is %q in aml-serve@.service and %q in aml-watch@.service, want one value in both", key, want, watch["Service"][key]) + } + } + + timer := readUnit(t, "aml-watch@.timer") + for _, s := range []struct{ section, key, want string }{ + {"Timer", "OnBootSec", "2min"}, + {"Timer", "OnUnitActiveSec", "1min"}, + {"Timer", "AccuracySec", "1s"}, + {"Install", "WantedBy", "timers.target"}, + } { + if got := timer.get(t, s.section, s.key); got != s.want { + t.Errorf("aml-watch@.timer [%s] %s=%s, want %s", s.section, s.key, got, s.want) + } + } + + // The inspect service: inspect.env, as mnemon-instance inspect enable + // writes it, with only its own names. + inspectEnv := map[string]string{ + "AML_INSPECT_LISTEN": "10.0.0.5:8092", "AML_INSPECT_DEBUG_CAPS": "deepseek=5,jev=1", "MNEMON_INSPECT_TOKEN": "mnm_inspect-token", + "GOMEMLIMIT": "819MiB", "DEEPSEEK_API_KEY": "sk-deepseek", "TYPESAFE_API_KEY": "ts-jev", + } + inspect := readUnit(t, "aml-inspect@.service") + args = execArgs(t, inspect.get(t, "Service", "ExecStart"), instance, inspectEnv) + if len(args) < 3 || !slices.Equal(args[:3], []string{"/usr/local/bin/mnemon", "memory-agent", "aml-inspect"}) { + t.Fatal("aml-inspect@.service runs", args) + } + command = newAMLInspectCommand() + if err := command.ParseFlags(args[3:]); err != nil { + t.Fatal(err) + } + if rest := command.Flags().Args(); len(rest) > 0 { + t.Fatal("aml-inspect@.service passes arguments that are not flags:", rest) + } + for flag, want := range map[string]string{ + "listen": "10.0.0.5:8092", + "token-env": "MNEMON_INSPECT_TOKEN", + "data-root": "/var/lib/mnemon-aml/eval/data", + "memory-config": "/etc/mnemon-aml/eval/memory-config.json", + "debug-budget-file": "/var/lib/mnemon-aml-inspect/eval/debug-budget.db", + "debug-caps": "deepseek=5,jev=1", + "max-concurrent": "2", + "log-format": "json", + "provider-keys-file": "/var/lib/mnemon-aml/eval/provider-keys.json", + "settings-file": "/var/lib/mnemon-aml-inspect/eval/settings.json", + } { + if f := command.Flags().Lookup(flag); !f.Changed || f.Value.String() != want { + t.Errorf("aml-inspect@.service passes --%s %q, want %q", flag, f.Value.String(), want) + } + } + // It reads the instance's key file under the instance's state, which the + // unit mounts read only; its own settings file, which it writes, is in + // its own state, beside its debug budget. + if keys := command.Flags().Lookup("provider-keys-file").Value.String(); !strings.HasPrefix(keys, strings.ReplaceAll(inspect.get(t, "Service", "ReadOnlyPaths"), "%i", instance)+"/") { + t.Errorf("--provider-keys-file %s is not under ReadOnlyPaths=", keys) + } + if settings, debug := command.Flags().Lookup("settings-file").Value.String(), command.Flags().Lookup("debug-budget-file").Value.String(); path.Dir(settings) != path.Dir(debug) || + path.Dir(settings) != "/var/lib/"+strings.ReplaceAll(inspect.get(t, "Service", "StateDirectory"), "%i", instance) { + t.Errorf("--settings-file %s is not in the service's state directory", settings) + } + // The units run on Linux: on Windows their paths are not absolute (R89), + // so there the flags are parsed but not checked. + if runtime.GOOS != "windows" { + t.Setenv("MNEMON_INSPECT_TOKEN", strings.Repeat("i", 47)) + options, err := amlInspectFlags(command) + if err != nil || options.caps["deepseek"] != 5 || options.caps["jev"] != 1 { + t.Fatal("aml-inspect@.service's flags do not parse:", err) + } + } + debugBudget, dataRoot := command.Flags().Lookup("debug-budget-file").Value.String(), command.Flags().Lookup("data-root").Value.String() + // The debug budget lives in the service's own state, apart from the + // instance's budget file (memory-config.json's budgetFile). + if dir := path.Dir(debugBudget); dir != "/var/lib/mnemon-aml-inspect/eval" || debugBudget == "/var/lib/mnemon-aml/eval/model-budget.db" { + t.Errorf("--debug-budget-file %q is not in the service's state directory", debugBudget) + } + for _, s := range []struct{ section, key, want string }{ + {"Service", "Type", "simple"}, + {"Service", "User", "mnemon"}, + {"Service", "Group", "mnemon"}, + {"Service", "Restart", "on-failure"}, + {"Service", "EnvironmentFile", "/etc/mnemon-aml/%i/inspect.env"}, + // The instance's state (data root and budget file) is mounted read only. + {"Service", "ReadOnlyPaths", "/var/lib/mnemon-aml/%i"}, + {"Service", "StateDirectory", "mnemon-aml-inspect/%i"}, + {"Service", "StateDirectoryMode", "0700"}, + {"Service", "MemoryMax", "1G"}, + {"Service", "CPUQuota", "100%"}, + {"Service", "Nice", "10"}, + // Best-effort at its lowest priority, not idle (R80): the idle + // class can starve under sustained I/O, and an inspect read + // holding its SHARED lock on the rollback-journal memory-index.db + // would then hold up aml-serve's consolidation writes. + {"Service", "IOSchedulingClass", "best-effort"}, + {"Service", "IOSchedulingPriority", "7"}, + {"Install", "WantedBy", "multi-user.target"}, + } { + if got := inspect.get(t, s.section, s.key); got != s.want { + t.Errorf("aml-inspect@.service [%s] %s=%s, want %s", s.section, s.key, got, s.want) + } + } + // aml-inspect's shutdown (aml_inspect.go) fits in its TimeoutStopSec as + // aml-serve's does (Minor 2, R90). + if stop, shutdown := systemdSpan(t, inspect.get(t, "Service", "TimeoutStopSec")), inspectShutdownGrace+inspectCancelGrace+inspectSettleGrace; stop <= shutdown { + t.Errorf("aml-inspect@.service TimeoutStopSec %s does not outlast the shutdown's %s", stop, shutdown) + } + if !strings.HasPrefix(dataRoot, strings.ReplaceAll(inspect.get(t, "Service", "ReadOnlyPaths"), "%i", instance)+"/") { + t.Errorf("--data-root %s is not under ReadOnlyPaths=", dataRoot) + } + if requires, ok := inspect["Unit"]["Requires"]; ok { + t.Errorf("aml-inspect@.service requires %q: it reads stores whether aml-serve runs or not", requires) + } + for _, key := range unitSandbox { + if want := serve["Service"][key]; len(want) == 0 || !slices.Equal(inspect["Service"][key], want) { + t.Errorf("%s is %q in aml-serve@.service and %q in aml-inspect@.service, want one value in both", key, want, inspect["Service"][key]) + } + } + // The inspect port is the admin port + 1, inside the firewall's + // 8091–8600 overlay rule. + _, port, _ := strings.Cut(inspectEnv["AML_INSPECT_LISTEN"], ":") + _, adminPort, _ := strings.Cut(env["AML_ADMIN_LISTEN"], ":") + if p, a := atoiPort(t, port), atoiPort(t, adminPort); p != a+1 || p < 8091 || p > 8600 { + t.Errorf("inspect port %d, admin port %d", p, a) + } +} + +func atoiPort(t *testing.T, port string) int { + t.Helper() + n := 0 + for _, c := range port { + if c < '0' || c > '9' { + t.Fatalf("port %q", port) + } + n = n*10 + int(c-'0') + } + return n +} + +// The Caddyfiles must proxy to the ports the instances listen on, hold +// requests through a restart, and keep the platform's limits. The shared +// host's snippets are the instance script's (test/deploy). +func TestDeployKitMatchesTheCommand(t *testing.T) { + read := func(name string) string { + raw, err := os.ReadFile(deployFile(t, name)) + if err != nil { + t.Fatal(err) + } + // Compare without the comments and with the column alignment collapsed. + var kept []string + for _, line := range strings.Split(string(raw), "\n") { + if before, _, found := strings.Cut(line, "#"); found { + line = before + } + kept = append(kept, line) + } + return strings.Join(strings.Fields(strings.Join(kept, "\n")), " ") + } + // The platform waits up to 31 minutes for an Add or a Search, and + // aml-serve takes bodies of up to 30 MiB. + limits := []string{"read_body 35m", "read_header 30s", "write 35m", "idle 5m", "max_size 30MiB"} + + dedicated := read("Caddyfile.dedicated") + // Slot 0 of a dedicated host: data 127.0.0.1:8090. Caddy dials again + // through a restart, and its upstream keep-alive stays below aml-serve's + // 2-minute idle timeout. + for _, want := range append([]string{ + "reverse_proxy 127.0.0.1:8090 {", "lb_try_duration 30s", "lb_try_interval 250ms", + "keepalive 60s", "dial_timeout 5s", "response_header_timeout 35m", "read_timeout 35m", "write_timeout 35m", + }, limits...) { + if !strings.Contains(dedicated, want) { + t.Errorf("Caddyfile.dedicated lacks %q", want) + } + } + + shared := read("Caddyfile.shared") + for _, want := range append([]string{ + "import /etc/caddy/mnemon-instances/*.caddy", + "handle { header Content-Type application/json respond `{\"error\":\"not found\"}` 404 }", + }, limits...) { + if !strings.Contains(shared, want) { + t.Errorf("Caddyfile.shared lacks %q", want) + } + } + // A shared host reaches its instances through their snippets only. + if strings.Contains(shared, "reverse_proxy") { + t.Error("Caddyfile.shared proxies outside the instances' snippets") + } +} + +// syncLog is a log destination the command and the test share. +type syncLog struct { + mu sync.Mutex + b strings.Builder +} + +func (l *syncLog) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.Write(p) +} + +func (l *syncLog) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.String() +} + +// waitFor polls done for up to ten seconds. +func waitFor(t *testing.T, what string, done func() bool) { + t.Helper() + for start := time.Now(); !done(); time.Sleep(5 * time.Millisecond) { + if time.Since(start) > 10*time.Second { + t.Fatal("timed out waiting for", what) + } + } +} + +// notifications records, in order, what the command tells systemd. +type notifications struct { + mu sync.Mutex + states []string +} + +func (n *notifications) all() []string { + n.mu.Lock() + defer n.mu.Unlock() + return slices.Clone(n.states) +} + +// recordNotifications replaces sd_notify until the test ends. +func recordNotifications(t *testing.T) *notifications { + t.Helper() + n := ¬ifications{} + previous := sdNotify + sdNotify = func(state string) error { + n.mu.Lock() + n.states = append(n.states, state) + n.mu.Unlock() + return nil + } + t.Cleanup(func() { sdNotify = previous }) + return n +} + +// shortGrace makes the shutdown's windows short until the test ends. +func shortGrace(t *testing.T, grace, cancel time.Duration) { + t.Helper() + previousGrace, previousCancel := amlShutdownGrace, amlCancelGrace + amlShutdownGrace, amlCancelGrace = grace, cancel + t.Cleanup(func() { amlShutdownGrace, amlCancelGrace = previousGrace, previousCancel }) +} + +// modelEndpoint stands in for DeepSeek and JEV (their testEndpoint): every +// request it reads waits until its client leaves or the endpoint is released, +// then is answered 503. +type modelEndpoint struct { + *httptest.Server + requests atomic.Int64 + release chan struct{} + once sync.Once +} + +func newModelEndpoint(t *testing.T) *modelEndpoint { + t.Helper() + e := &modelEndpoint{release: make(chan struct{})} + e.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.Copy(io.Discard, r.Body) + e.requests.Add(1) + select { + case <-r.Context().Done(): + case <-e.release: + } + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(func() { + e.free() + e.Close() + }) + return e +} + +// free answers the waiting requests and those to come. +func (e *modelEndpoint) free() { e.once.Do(func() { close(e.release) }) } + +// amlTestConfig writes a memory configuration whose planner (DeepSeek) and +// judge (JEV), and consolidator when edit adds one, call endpoint; the +// budget file is the test's own. +func amlTestConfig(t *testing.T, endpoint string, edit func(*memoryagent.Config)) string { + t.Helper() + t.Setenv("AML_TEST_DEEPSEEK", "test-only-placeholder") + t.Setenv("AML_TEST_JEV", "test-only-placeholder") + dir := t.TempDir() + config := memoryagent.Config{ + Version: 1, + Strategy: "simple", + View: &memoryagent.ViewConfig{MaxItems: 100, MaxCharacters: 240000, Order: "time"}, + Planner: &memoryagent.ModelConfig{Kind: "deepseek", Model: "deepseek-flash", KeyEnv: "AML_TEST_DEEPSEEK", TestEndpoint: endpoint}, + Judge: &memoryagent.ModelConfig{Kind: "jev", Model: "jev-1.13.0", KeyEnv: "AML_TEST_JEV", TestEndpoint: endpoint}, + BudgetFile: filepath.Join(dir, "model-budget.db"), + } + if edit != nil { + edit(&config) + } + raw, err := json.Marshal(config) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "memory-config.json") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + return path +} + +// amlServeRun is an aml-serve command the test runs until it stops it. +type amlServeRun struct { + root string + log *syncLog + cancel context.CancelFunc + done chan error + once sync.Once + err error +} + +// startAMLServe runs aml-serve on a loopback port of its own, over a new +// data root, with configPath and args; the test's end stops it. +func startAMLServe(t *testing.T, configPath string, args ...string) *amlServeRun { + t.Helper() + run := newAMLServeRun(t) + run.start(t, configPath, args...) + return run +} + +// newAMLServeRun is an aml-serve run not started yet: its data root and log +// exist. +func newAMLServeRun(t *testing.T) *amlServeRun { + t.Helper() + return &amlServeRun{root: t.TempDir(), log: &syncLog{}, done: make(chan error, 1)} +} + +// start starts the run's command; the test's end stops it. +func (run *amlServeRun) start(t *testing.T, configPath string, args ...string) { + t.Helper() + t.Setenv("AML_TEST_KEY", testAMLKey) + command := newAMLServeCommand() + command.SetArgs(append([]string{"--listen", "127.0.0.1:0", "--data-root", run.root, "--memory-config", configPath, "--auth-key-env", "AML_TEST_KEY"}, args...)) + command.SilenceUsage, command.SilenceErrors = true, true + command.SetOut(io.Discard) + command.SetErr(run.log) + ctx, cancel := context.WithCancel(context.Background()) + run.cancel = cancel + go func() { run.done <- command.ExecuteContext(ctx) }() + t.Cleanup(func() { run.stop(t) }) +} + +const testAMLKey = "test-aml-key" + +// stop stops the command as SIGTERM does and returns its error. +func (r *amlServeRun) stop(t *testing.T) error { + t.Helper() + r.once.Do(func() { + r.cancel() + select { + case r.err = <-r.done: + case <-time.After(30 * time.Second): + t.Fatal("aml-serve did not stop") + } + }) + return r.err +} + +var listeningOn = regexp.MustCompile(`aml-serve (admin )?listening on (127\.0\.0\.1:\d+)`) + +// address is the address the command's public (or admin) listener is bound +// to, as it logged it. +func (r *amlServeRun) address(t *testing.T, admin bool) string { + t.Helper() + var addr string + waitFor(t, "the listening line", func() bool { + for _, match := range listeningOn.FindAllStringSubmatch(r.log.String(), -1) { + if (match[1] != "") == admin { + addr = match[2] + } + } + return addr != "" + }) + return "http://" + addr +} + +// post sends body to the public listener with the API key. It waits for the +// listening line, which fails the test when it never comes: call it on the +// test's goroutine only, and postTo on any other (the final review's parked +// T9-g). +func (r *amlServeRun) post(t *testing.T, path string, body any) (int, string) { + t.Helper() + return postTo(r.address(t, false), path, body) +} + +// postTo sends body to base+path with the API key. It fails no test, so any +// goroutine may call it: a request that cannot be made or sent answers 0, +// with its error as the body. +func postTo(base, path string, body any) (int, string) { + raw, err := json.Marshal(body) + if err != nil { + return 0, err.Error() + } + request, err := http.NewRequest(http.MethodPost, base+path, bytes.NewReader(raw)) + if err != nil { + return 0, err.Error() + } + request.Header.Set("Authorization", "Bearer "+testAMLKey) + response, err := (&http.Client{Timeout: 30 * time.Second}).Do(request) + if err != nil { + return 0, err.Error() + } + defer response.Body.Close() + answer, _ := io.ReadAll(response.Body) + return response.StatusCode, string(answer) +} + +// get fetches url and returns its status. +func get(t *testing.T, url string) int { + t.Helper() + response, err := (&http.Client{Timeout: 10 * time.Second}).Get(url) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + io.Copy(io.Discard, response.Body) + return response.StatusCode +} + +func noteAdd(requestID, userID string) aml.AddRequest { + return aml.AddRequest{RequestID: requestID, UserID: userID, SessionID: "s1", Messages: []aml.Message{ + {Role: "user", Content: "I went to the LGBTQ support group yesterday, Caroline here; everyone shared their stories and it was powerful."}, + {Role: "assistant", Content: "That sounds meaningful. How did it feel to be there with the others?"}, + }} +} + +func searchRequest(userID string) aml.SearchRequest { + return aml.SearchRequest{Query: "When did Caroline go to the LGBTQ support group?", UserID: userID, TopK: 10} +} + +// healthAt is the status the public /health answers within a second, at the +// address the log names; 0 when it does not answer. +func healthAt(log string) int { + for _, match := range listeningOn.FindAllStringSubmatch(log, -1) { + if match[1] != "" { + continue + } + response, err := (&http.Client{Timeout: time.Second}).Get("http://" + match[2] + "/health") + if err != nil { + return 0 + } + response.Body.Close() + return response.StatusCode + } + return 0 +} + +// lockFree reports whether the data root's lock is free: a test can take it. +func lockFree(t *testing.T, root string) bool { + t.Helper() + lock := flock.New(filepath.Join(root, ".aml-serve.lock")) + locked, err := lock.TryLock() + if err != nil { + t.Fatal(err) + } + if locked { + lock.Unlock() + } + return locked +} + +// The process starts its components in order and tells systemd READY=1 once +// the public listener serves, and the admin listener too when its address +// bound (R91); a shutdown with nothing in flight tells it STOPPING=1, closes +// everything and exits 0, giving the lock back. +func TestAMLServeServesUntilStopped(t *testing.T) { + notified := recordNotifications(t) + // When READY=1 is sent, the public server already answers. + run := newAMLServeRun(t) + readyHealth := make(chan int, 1) + record := sdNotify + sdNotify = func(state string) error { + if state == "READY=1" { + readyHealth <- healthAt(run.log.String()) + } + return record(state) + } + endpoint := newModelEndpoint(t) + run.start(t, amlTestConfig(t, endpoint.URL, nil), "--admin-listen", "127.0.0.1:0", "--log-format", "json") + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + if code := <-readyHealth; code != http.StatusOK { + t.Fatal("public /health when READY=1 was sent:", code) + } + public, admin := run.address(t, false), run.address(t, true) + if public == admin { + t.Fatal("one listener for both", public) + } + if code := get(t, public+"/health"); code != http.StatusOK { + t.Fatal("public /health", code) + } + for _, path := range []string{"/metrics", "/admin/status"} { + if code := get(t, admin+path); code != http.StatusOK { + t.Fatal("admin", path, code) + } + } + if code := get(t, public+"/metrics"); code != http.StatusNotFound { + t.Fatal("the public listener serves /metrics", code) + } + if code, body := run.post(t, "/add", noteAdd("r1", "u1")); code != http.StatusOK { + t.Fatal(code, body) + } + if lockFree(t, run.root) { + t.Fatal("the data root is not locked while serving") + } + if err := run.stop(t); err != nil { + t.Fatal(err, run.log.String()) + } + if got := notified.all(); !slices.Equal(got, []string{"READY=1", "STOPPING=1"}) { + t.Fatal(got) + } + if !lockFree(t, run.root) { + t.Fatal("the lock was not given back") + } + // --log-format json: every line is a JSON object. + for _, line := range strings.Split(strings.TrimSpace(run.log.String()), "\n") { + if !json.Valid([]byte(line)) { + t.Fatal("not JSON:", line) + } + } + if strings.Contains(run.log.String(), "cancel") || endpoint.requests.Load() != 0 { + t.Fatal(endpoint.requests.Load(), run.log.String()) + } +} + +// getWith fetches url with token as its bearer token (none when empty) and +// returns its status and body. +func getWith(t *testing.T, url, token string) (int, string) { + t.Helper() + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := (&http.Client{Timeout: 10 * time.Second}).Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + body, _ := io.ReadAll(response.Body) + return response.StatusCode, string(body) +} + +// With --admin-token-env the admin listener answers only the requests that +// hold the token (the profiler and an unknown path included), tells which +// build runs, and the token is nowhere in the command's log. The public +// listener does not notice. +func TestAMLServeAdminToken(t *testing.T) { + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + const token = "aml-serve-e2e-admin-token-0123456789" + t.Setenv("AML_TEST_ADMIN_TOKEN", token) + run := startAMLServe(t, amlTestConfig(t, endpoint.URL, nil), "--admin-listen", "127.0.0.1:0", "--admin-token-env", "AML_TEST_ADMIN_TOKEN") + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + public, admin := run.address(t, false), run.address(t, true) + + for _, path := range []string{"/metrics", "/ready", "/admin/status", "/admin/v1/stores", "/debug/pprof/", "/no-such-route"} { + for _, sent := range []string{"", "wrong-" + token, token[:len(token)-1]} { + if code, body := getWith(t, admin+path, sent); code != http.StatusUnauthorized || !strings.Contains(body, "missing or invalid admin token") { + t.Errorf("%s with token %q: %d %s", path, sent, code, body) + } + } + } + for _, path := range []string{"/metrics", "/admin/status", "/admin/v1/stores", "/debug/pprof/"} { + if code, body := getWith(t, admin+path, token); code != http.StatusOK { + t.Errorf("%s with the token: %d %.80s", path, code, body) + } + } + // /ready says yes or no by the disk, but never 401. + if code, _ := getWith(t, admin+"/ready", token); code != http.StatusOK && code != http.StatusServiceUnavailable { + t.Errorf("/ready with the token: %d", code) + } + _, body := getWith(t, admin+"/admin/status", token) + var status aml.AdminStatus + if err := json.Unmarshal([]byte(body), &status); err != nil || status.Version != version { + t.Errorf("/admin/status says version %q, want %q: %v", status.Version, version, err) + } + _, body = getWith(t, admin+"/metrics", token) + if !strings.Contains(body, "mnemon_aml_admin_auth_failures_total 18\n") { + t.Error("the failures were not counted:", body) + } + + // The data plane is as it was: no token on the public listener. + if code := get(t, public+"/health"); code != http.StatusOK { + t.Error("public /health", code) + } + if code, body := run.post(t, "/add", noteAdd("r1", "u1")); code != http.StatusOK { + t.Error("public /add", code, body) + } + if err := run.stop(t); err != nil { + t.Fatal(err, run.log.String()) + } + if strings.Contains(run.log.String(), token) { + t.Error("the log shows the admin token:", run.log.String()) + } +} + +// A Search still running when the grace ends is canceled: it answers 503, +// which the platform retries, and the command fails (exit status 1), saying +// what it cut. +func TestShutdownExitsNonZeroWhenRequestsCancelled(t *testing.T) { + shortGrace(t, 200*time.Millisecond, 5*time.Second) + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + run := startAMLServe(t, amlTestConfig(t, endpoint.URL, nil)) + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + if code, body := run.post(t, "/add", noteAdd("r1", "u1")); code != http.StatusOK { + t.Fatal(code, body) + } + type answer struct { + code int + body string + } + searched := make(chan answer, 1) + public := run.address(t, false) // on the test's goroutine: it may fail the test + go func() { + code, body := postTo(public, "/search", searchRequest("u1")) + searched <- answer{code, body} + }() + // The Search waits on its planner. + waitFor(t, "the planner's request", func() bool { return endpoint.requests.Load() > 0 }) + start := time.Now() + err := run.stop(t) + // It says how many it canceled (the final review's parked T9-e). + if err == nil || !strings.Contains(err.Error(), "canceled at the end of the 200ms grace: 1 request still running") { + t.Fatal(err, run.log.String()) + } + if elapsed := time.Since(start); elapsed < 200*time.Millisecond || elapsed > 5*time.Second { + t.Fatal("stopped after", elapsed) + } + if got := <-searched; got.code != http.StatusServiceUnavailable || !strings.Contains(got.body, "request canceled") { + t.Fatal(got) + } + if !strings.Contains(run.log.String(), "the 200ms grace has ended with 1 request still running; canceling them") { + t.Fatal(run.log.String()) + } + if got := notified.all(); !slices.Equal(got, []string{"READY=1", "STOPPING=1"}) { + t.Fatal(got) + } + if !lockFree(t, run.root) { + t.Fatal("the lock was not given back") + } +} + +// A background consolidation job still running when the grace ends is cut, +// and the command fails, saying so; the job's user resumes after a restart. +func TestShutdownCutsConsolidationJobsPastTheGrace(t *testing.T) { + shortGrace(t, 200*time.Millisecond, 5*time.Second) + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + config := amlTestConfig(t, endpoint.URL, func(config *memoryagent.Config) { + config.Consolidation = &memoryagent.ConsolidationConfig{Mode: "timelines", BatchCharacters: 300} + config.Consolidator = &memoryagent.ModelConfig{Kind: "deepseek", Model: "deepseek-flash", KeyEnv: "AML_TEST_DEEPSEEK", TestEndpoint: endpoint.URL} + }) + run := startAMLServe(t, config, "--consolidate", "background", "--consolidation-workers", "1") + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + for i := range 4 { + if code, body := run.post(t, "/add", noteAdd(fmt.Sprintf("r%d", i), "u1")); code != http.StatusOK { + t.Fatal(code, body) + } + } + // A worker's job waits on the consolidator. + waitFor(t, "the consolidator's request", func() bool { return endpoint.requests.Load() > 0 }) + err := run.stop(t) + if err == nil || !strings.Contains(err.Error(), "cut at the end of the 200ms grace: consolidation jobs still running: 1") || strings.Contains(err.Error(), "requests") { + t.Fatal(err, run.log.String()) + } + if !strings.Contains(run.log.String(), "the 200ms grace has ended with consolidation jobs still running: 1; cutting them: their saved batches stay, and they resume after a restart") { + t.Fatal(run.log.String()) + } + if !lockFree(t, run.root) { + t.Fatal("the lock was not given back") + } +} + +// The scheduler drains as the shutdown starts: a background Add waiting at +// the lag bound answers 200 at once (its records are stored), not at the end +// of the grace, and is not counted as canceled; the job is cut later. +func TestShutdownAnswersAddsWaitingAtTheBound(t *testing.T) { + shortGrace(t, 2*time.Second, 5*time.Second) + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + // A job's window is one batch of 300 characters, the lag bound. + config := amlTestConfig(t, endpoint.URL, func(config *memoryagent.Config) { + config.Consolidation = &memoryagent.ConsolidationConfig{Mode: "timelines", BatchCharacters: 300, MaxBatches: 1} + config.Consolidator = &memoryagent.ModelConfig{Kind: "deepseek", Model: "deepseek-flash", KeyEnv: "AML_TEST_DEEPSEEK", TestEndpoint: endpoint.URL} + }) + run := startAMLServe(t, config, "--consolidate", "background") + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + if code, body := run.post(t, "/add", noteAdd("r0", "u1")); code != http.StatusOK { + t.Fatal(code, body) + } + // Past the bound, the next Add waits for the job, whose fold waits at + // the endpoint. + type answer struct { + code int + at time.Time + } + added := make(chan answer, 1) + public := run.address(t, false) // on the test's goroutine: it may fail the test + go func() { + code, _ := postTo(public, "/add", noteAdd("r1", "u1")) + added <- answer{code, time.Now()} + }() + waitFor(t, "the consolidator's request", func() bool { return endpoint.requests.Load() > 0 }) + select { + case got := <-added: + t.Fatal("the Add did not wait at the bound", got.code) + case <-time.After(100 * time.Millisecond): + } + start := time.Now() + err := run.stop(t) + got := <-added + if got.code != http.StatusOK || got.at.Sub(start) > time.Second { + t.Fatal("the waiting Add answered", got.code, "after", got.at.Sub(start)) + } + if err == nil || !strings.Contains(err.Error(), "consolidation jobs still running: 1") || strings.Contains(err.Error(), "requests") { + t.Fatal(err, run.log.String()) + } +} + +// The supervisor starts a component only once the one before is ready, fails +// one that returns before it is ready, ends failed at the first failure, and +// waits for the components until a deadline. +func TestSupervisorStartsInOrder(t *testing.T) { + var mu sync.Mutex + order := []string{} + note := func(event string) { + mu.Lock() + order = append(order, event) + mu.Unlock() + } + sup := newSupervisor() + release := make(chan struct{}) + serving := func(name string, delay time.Duration) func(ready func()) error { + return func(ready func()) error { + time.Sleep(delay) + note(name + " ready") + ready() + <-release + return nil + } + } + if err := sup.start("first", serving("first", 20*time.Millisecond)); err != nil { + t.Fatal(err) + } + note("first started") + if err := sup.start("second", serving("second", 0)); err != nil { + t.Fatal(err) + } + mu.Lock() + if !slices.Equal(order, []string{"first ready", "first started", "second ready"}) { + t.Fatal(order) + } + mu.Unlock() + if sup.failed.Err() != nil || sup.err() != nil { + t.Fatal("failed before any failure", sup.err()) + } + if err := sup.start("early", func(func()) error { return nil }); err == nil || err.Error() != "early: it stopped before it was ready" { + t.Fatal(err) + } + if err := sup.start("broken", func(func()) error { return errors.New("address in use") }); err == nil || err.Error() != "broken: address in use" { + t.Fatal(err) + } + select { + case <-sup.failed.Done(): + case <-time.After(5 * time.Second): + t.Fatal("a failure did not end failed") + } + if err := sup.err(); err == nil || err.Error() != "early: it stopped before it was ready" { + t.Fatal("the first failure", err) + } + if stuck := sup.wait(time.Now().Add(20 * time.Millisecond)); !slices.Equal(stuck, []string{"first", "second"}) { + t.Fatal(stuck) + } + close(release) + if stuck := sup.wait(time.Now().Add(5 * time.Second)); stuck != nil { + t.Fatal(stuck) + } +} + +// R90: closing the models waits for their requests in flight, abandoned ones +// included, until the end of the shutdown's windows, then cuts those still +// running. Each settles its reservation at its input estimate before the +// command returns, within a few seconds of the windows (TimeoutStopSec leaves +// it more), non-zero, saying what it cut: nothing stays held, and the budget +// file is closed. How many it cuts, one or two, is not the test's to say +// (the re-review of the final fix's N1): see the comment at the Search. +func TestShutdownCutsAbandonedModelRequests(t *testing.T) { + shortGrace(t, 100*time.Millisecond, 400*time.Millisecond) + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + // A planner attempt stops waiting after one second and is not retried; + // its request runs on toward the minute's hard timeout. + configPath := amlTestConfig(t, endpoint.URL, func(config *memoryagent.Config) { + config.Planner.TimeoutSeconds, config.Planner.AttemptTimeoutSeconds, config.Planner.MaxAttempts = 60, 1, 1 + }) + run := startAMLServe(t, configPath) + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + if code, body := run.post(t, "/add", noteAdd("r1", "u1")); code != http.StatusOK { + t.Fatal(code, body) + } + // The planner fails, so the Search answers the raw ranking. Its two + // planner calls, the cue plan and the need plan, send their requests at + // once, and they cancel each other when one fails (simple_plan.go, plan). + // When the first attempt's second is up, the other call may still be + // waiting: it is canceled with its plan, cuts its own request and settles + // it at its estimate before the shutdown, so one request is left + // abandoned. If its own second is up first, both are, waiting at the + // endpoint. Which it is, is a race between two timers, so the close cuts + // one request or two. + if code, body := run.post(t, "/search", searchRequest("u1")); code != http.StatusOK { + t.Fatal(code, body) + } + if n := endpoint.requests.Load(); n != 2 { + t.Fatal("planner requests", n) + } + start := time.Now() + err := run.stop(t) + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Fatal("the shutdown waited for the abandoned requests", elapsed) + } + const cut = `deepseek: cut [12] model requests? still in flight at the close's deadline: their reservations settled at their input estimates` + if err == nil || !regexp.MustCompile(cut).MatchString(err.Error()) || strings.Contains(err.Error(), "did not finish") || strings.Contains(err.Error(), "requests still running") { + t.Fatal(err, run.log.String()) + } + if !regexp.MustCompile("aml-serve: " + cut).MatchString(run.log.String()) { + t.Fatal(run.log.String()) + } + if held, spent := budgetAccount(t, configPath, "deepseek"); held != 0 || spent <= 0 { + t.Fatal("deepseek held", held, "spent", spent) + } +} + +// manualWatchdogTicker makes the watchdog's ticker one the test ticks +// itself, until the test ends: ticks are taken from the returned channel, a +// send returning once the watchdog has taken the tick; period is what the +// watchdog asked for, and stopped closes when it stops its ticker. +func manualWatchdogTicker(t *testing.T) (ticks chan time.Time, period *atomic.Int64, stopped chan struct{}) { + t.Helper() + ticks, period, stopped = make(chan time.Time), &atomic.Int64{}, make(chan struct{}) + previous := amlWatchdogTicker + amlWatchdogTicker = func(d time.Duration) (<-chan time.Time, func()) { + period.Store(int64(d)) + return ticks, sync.OnceFunc(func() { close(stopped) }) + } + t.Cleanup(func() { amlWatchdogTicker = previous }) + return ticks, period, stopped +} + +// The watchdog pings at each tick of its ticker, every half interval, while +// the liveness probe answers within a quarter interval; it skips the ping, +// and logs why, while the probe fails; and it stops its ticker and returns +// with its context. The test ticks the ticker itself (the final review's +// parked T9-b): no wall-clock cadence to miss under load. +func TestWatchdogPingsOnlyWhileAlive(t *testing.T) { + const interval = 40 * time.Millisecond + ticks, period, stopped := manualWatchdogTicker(t) + var alive atomic.Bool + alive.Store(true) + pings := make(chan string, 100) + log := &syncLog{} + probe := func(ctx context.Context) error { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > interval/4 { + t.Error("a probe without its quarter-interval deadline") + } + if !alive.Load() { + return errors.New("stuck") + } + return nil + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + go func() { + amlWatchdog(ctx, interval, probe, func(state string) { pings <- state }, slog.New(slog.NewTextHandler(log, nil))) + close(done) + }() + ping := func(what string) { + t.Helper() + select { + case state := <-pings: + if state != "WATCHDOG=1" { + t.Fatal(what, state) + } + case <-time.After(5 * time.Second): + t.Fatal(what, "no ping") + } + } + ticks <- time.Now() + ping("the first tick:") + if got := time.Duration(period.Load()); got != interval/2 { + t.Fatal("the watchdog ticks every", got, "want", interval/2) + } + // While the probe fails: no ping. A tick is taken only once the one + // before was handled, so after two ticks the first is done. + alive.Store(false) + ticks <- time.Now() + ticks <- time.Now() + if len(pings) != 0 || !strings.Contains(log.String(), "aml-serve watchdog: no ping: stuck") { + t.Fatal("pings while the probe fails:", len(pings), log.String()) + } + alive.Store(true) + ticks <- time.Now() + ping("once the probe passes again:") + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the watchdog did not stop") + } + select { + case <-stopped: + default: + t.Fatal("the watchdog's ticker was not stopped") + } +} + +// Parked T9-a (final review): the watchdog stops, and has stopped, before +// systemd is told STOPPING=1, so no ping can follow it: when STOPPING=1 is +// sent, the watchdog takes no tick any more. +func TestWatchdogStopsBeforeStopping(t *testing.T) { + ticks, _, _ := manualWatchdogTicker(t) + t.Setenv("WATCHDOG_USEC", "2000000") + var mu sync.Mutex + states := []string{} + tickedAtStopping := false + previous := sdNotify + sdNotify = func(state string) error { + mu.Lock() + states = append(states, state) + mu.Unlock() + if state == "STOPPING=1" { + select { + case ticks <- time.Now(): // the watchdog still runs: its ping follows + tickedAtStopping = true + time.Sleep(200 * time.Millisecond) + case <-time.After(200 * time.Millisecond): + } + } + return nil + } + t.Cleanup(func() { sdNotify = previous }) + notified := func() []string { + mu.Lock() + defer mu.Unlock() + return slices.Clone(states) + } + endpoint := newModelEndpoint(t) + run := startAMLServe(t, amlTestConfig(t, endpoint.URL, nil)) + waitFor(t, "READY=1", func() bool { return slices.Contains(notified(), "READY=1") }) + ticks <- time.Now() + waitFor(t, "a ping", func() bool { return slices.Contains(notified(), "WATCHDOG=1") }) + if err := run.stop(t); err != nil { + t.Fatal(err, run.log.String()) + } + if got := notified(); tickedAtStopping || !slices.Equal(got, []string{"READY=1", "WATCHDOG=1", "STOPPING=1"}) { + t.Fatal("the watchdog ran when STOPPING=1 was sent:", tickedAtStopping, got) + } +} + +// The liveness probe passes while the public handler answers, Run's heartbeat +// is recent and the budget file answers; it touches no model. +func TestLivenessProbe(t *testing.T) { + endpoint := newModelEndpoint(t) + config, err := readMemoryConfig(amlTestConfig(t, endpoint.URL, nil)) + if err != nil { + t.Fatal(err) + } + server, err := aml.NewServer(aml.ServerConfig{DataRoot: t.TempDir(), MemoryConfig: config, AuthKey: "k", Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}) + if err != nil { + t.Fatal(err) + } + defer server.Close() + ctx := context.Background() + if err := amlLive(ctx, server, time.Minute); err == nil || !strings.Contains(err.Error(), "the background work has not ticked") { + t.Fatal(err) + } + runCtx, stopRun := context.WithCancel(ctx) + ready, done := make(chan struct{}), make(chan error, 1) + go func() { done <- server.Run(runCtx, func() { close(ready) }) }() + <-ready + if err := amlLive(ctx, server, time.Minute); err != nil { + t.Fatal(err) + } + // Once Run has stopped, the heartbeat ages past the interval. + stopRun() + if err := <-done; err != nil { + t.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + if err := amlLive(ctx, server, 10*time.Millisecond); err == nil || !strings.Contains(err.Error(), "the background work last ticked") { + t.Fatal(err) + } + // A closed budget file does not answer. + server.Close() + if err := amlLive(ctx, server, time.Minute); err == nil || !strings.Contains(err.Error(), "the budget file does not answer") { + t.Fatal(err) + } + if n := endpoint.requests.Load(); n != 0 { + t.Fatal("the probe called a model", n) + } +} + +// The admin listener's server times out slow clients but outlasts the +// profiles pprof serves. +func TestAdminServerTimeouts(t *testing.T) { + server := amlAdminServer(http.NotFoundHandler(), slog.New(slog.NewTextHandler(io.Discard, nil))) + if server.ReadHeaderTimeout != 10*time.Second || server.WriteTimeout != 5*time.Minute || server.IdleTimeout != 2*time.Minute { + t.Fatal(server.ReadHeaderTimeout, server.WriteTimeout, server.IdleTimeout) + } +} + +// With systemd (Type=notify, WatchdogSec), the process tells its socket +// READY=1, then WATCHDOG=1 every half interval, then STOPPING=1: the +// environment stays set for every notification. +func TestAMLServeNotifiesSystemd(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("systemd's notify socket is a Unix datagram socket") + } + dir, err := os.MkdirTemp("", "sd") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + socketPath := filepath.Join(dir, "notify") + socket, err := net.ListenUnixgram("unixgram", &net.UnixAddr{Name: socketPath, Net: "unixgram"}) + if err != nil { + t.Fatal(err) + } + defer socket.Close() + states := make(chan string, 100) + go func() { + buf := make([]byte, 256) + for { + n, err := socket.Read(buf) + if err != nil { + return + } + states <- string(buf[:n]) + } + }() + t.Setenv("NOTIFY_SOCKET", socketPath) + t.Setenv("WATCHDOG_USEC", "2000000") // the shortest interval it takes: a ping every second + endpoint := newModelEndpoint(t) + run := startAMLServe(t, amlTestConfig(t, endpoint.URL, nil)) + next := func() string { + select { + case state := <-states: + return state + case <-time.After(10 * time.Second): + t.Fatal("systemd was told nothing", run.log.String()) + return "" + } + } + if state := next(); state != "READY=1" { + t.Fatal(state) + } + for range 2 { + if state := next(); state != "WATCHDOG=1" { + t.Fatal(state) + } + } + if err := run.stop(t); err != nil { + t.Fatal(err) + } + for state := next(); state != "STOPPING=1"; state = next() { + if state != "WATCHDOG=1" { + t.Fatal(state) + } + } + select { + case state := <-states: + t.Fatal("told after STOPPING=1:", state) + case <-time.After(300 * time.Millisecond): + } +} + +// Parked T9-c (final review): a component that fails after READY=1 (here the +// public server, its listener closed under it) shuts the process down, which +// tells systemd STOPPING=1 and fails the command (exit status 1), naming the +// component. +func TestAMLServeFailsWhenAComponentFailsAfterReady(t *testing.T) { + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + config, err := readMemoryConfig(amlTestConfig(t, endpoint.URL, nil)) + if err != nil { + t.Fatal(err) + } + log := &syncLog{} + logger := slog.New(slog.NewTextHandler(log, nil)) + server, err := aml.NewServer(aml.ServerConfig{DataRoot: t.TempDir(), MemoryConfig: config, AuthKey: "k", Logger: logger}) + if err != nil { + t.Fatal(err) + } + process, err := newAMLProcess(server, amlServeOptions{listen: "127.0.0.1:0"}, logger, 0) + if err != nil { + server.Close() + t.Fatal(err) + } + served := make(chan error, 1) + go func() { served <- process.serve(context.Background(), func() {}) }() + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + process.publicListener.Close() + select { + case err = <-served: + case <-time.After(30 * time.Second): + t.Fatal("the process did not stop") + } + if err == nil || !strings.Contains(err.Error(), "public server: ") { + t.Fatal(err) + } + if !strings.Contains(log.String(), "aml-serve: public server: ") || !strings.Contains(log.String(), "; shutting down") { + t.Fatal(log.String()) + } + if got := notified.all(); !slices.Equal(got, []string{"READY=1", "STOPPING=1"}) { + t.Fatal(got) + } +} + +// Parked T9-c (final review): in background mode, a stop with nothing in +// flight drains the scheduler, closes everything and exits 0. +func TestAMLServeBackgroundModeStopsCleanly(t *testing.T) { + notified := recordNotifications(t) + endpoint := newModelEndpoint(t) + config := amlTestConfig(t, endpoint.URL, func(config *memoryagent.Config) { + config.Consolidation = &memoryagent.ConsolidationConfig{Mode: "timelines", BatchCharacters: 50000} + config.Consolidator = &memoryagent.ModelConfig{Kind: "deepseek", Model: "deepseek-flash", KeyEnv: "AML_TEST_DEEPSEEK", TestEndpoint: endpoint.URL} + }) + run := startAMLServe(t, config, "--consolidate", "background") + waitFor(t, "READY=1", func() bool { return slices.Contains(notified.all(), "READY=1") }) + if code, body := run.post(t, "/add", noteAdd("r1", "u1")); code != http.StatusOK { + t.Fatal(code, body) + } + if err := run.stop(t); err != nil { + t.Fatal(err, run.log.String()) + } + if got := notified.all(); !slices.Equal(got, []string{"READY=1", "STOPPING=1"}) { + t.Fatal(got) + } + if !lockFree(t, run.root) || endpoint.requests.Load() != 0 { + t.Fatal("the lock, or a model call:", lockFree(t, run.root), endpoint.requests.Load()) + } +} diff --git a/cmd/memory/aml_watch.go b/cmd/memory/aml_watch.go new file mode 100644 index 000000000..7c437be16 --- /dev/null +++ b/cmd/memory/aml_watch.go @@ -0,0 +1,1264 @@ +package memory + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "maps" + "math" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "syscall" + "time" + "unicode/utf8" + + "github.com/gofrs/flock" + "github.com/mnemon-dev/mnemon/internal/aml" + "github.com/mnemon-dev/mnemon/internal/memoryagent" + "github.com/spf13/cobra" +) + +// aml-watch is the alert watcher of an aml-serve: a one-shot command that a +// systemd timer runs every minute. One run +// +// 1. takes the exclusive lock .lock (see below); +// 2. reads the admin listener's /ready and /admin/status (with the admin +// token as a bearer token, when --admin-token-env names one) and the +// public listener's /health, each within 5 seconds; +// 3. applies the rules below to what it read and to the state the last run +// left (counters, when they last rose, when a condition began); +// 4. posts one text message per alert that fires, per firing alert that is +// due again, and per alert that clears, to the webhook in AML_ALERT_WEBHOOK +// (AML_ALERT_KIND: telegram, feishu or wecom), each within 10 seconds, each +// starting with the tag [mnemon-aml] (watchTag), whose header says which +// host the watcher runs on, and which instance with --instance; and +// 5. saves the state, atomically (a temporary file in the state's directory, +// renamed over it; mode 0600), whether or not a post failed. +// +// The admin token is a secret too. It is read once, from the variable +// --admin-token-env names, and sent to the admin listener only: not to the +// public one, and not along a redirect, whose Location is not the operator's. +// It is never printed, logged, put in an error, a message or the state. +// +// The webhook's URL is a secret (its path is a bearer token: a Feishu or WeCom +// hook's key, a Telegram bot's token): it is never printed, logged or put in +// an error. A failed post says the HTTP status, the code the webhook answered +// with, or that the webhook could not be reached (net/http's own error names +// the URL, so it is not used). A Telegram bot's URL is its sendMessage with the +// chat in the query (telegramTarget): the post goes to the URL without the +// query, the chat in its body, and only 2xx with "ok":true is a delivery +// (telegramRejection). A message that +// went out is recorded as sent, and not sent again as a new alert (unless the +// state cannot be saved: then the run exits non-zero and the next run says it +// again); one that did not go out stays unsent in the state, the next run +// sends it, and the run exits non-zero. Delivery is at-least-once, for every +// alert: an alert that never went out and then clears owes its resolution all +// the same, which says "alert not delivered" and is tried every run until the +// webhook takes it, with no cap on the tries and no age after which it is +// given up; the state keeps one entry per alert name. Without a webhook the +// alerts and their resolutions are logged instead (and kept, to be posted once +// there is a webhook) and the run exits zero, so that the timer can run before +// the webhook is configured. +// +// One run at a time: a run takes an exclusive, non-blocking lock on +// .lock (gofrs/flock; the file is left in place). If another run holds +// it, the run logs one line saying so, posts and saves nothing, and exits 0; +// two overlapping runs would post twice and the last to save would win. A lock +// that cannot be opened for another reason does not stop the run, which goes +// on without it, and says so. +// +// Rules, each an alert of the name shown (a name with a colon is per kind of +// model provider, or account): +// +// degraded-search a Search was answered by the raw ranking since the last run +// partial-search more than 5% of the last 10 minutes' Searches were partially +// degraded, once there were 20 of them +// breaker-open: a provider's circuit breaker is open, or opened since the last +// run (the count of trips only grows, so one that opened and +// recovered between two runs still shows) +// provider-rejected: a provider rejected requests with 401, 402 or 403 since the last +// run: a wrong or revoked key, or an empty balance (a status +// without rejectedByKind, an older server's, has nothing to say) +// budget-high: an account's spent and held amounts are above 80% of its cap +// budget-held: an account's held reservations are above 2% of its cap +// backlog-age the oldest work not yet consolidated has waited more than 15 minutes +// backlog-lag the largest user backlog has been above the server's lag bound +// for 15 minutes (the bound is read from the status) +// consolidation-dropped a batch was dropped from the consolidation layer +// link-fallback a batch was linked without JEV +// add-outran an Add waited out its window for consolidation +// restart the server started at another time than at the last run +// ready /ready did not answer 200, or /admin/status could not be read +// health the public /health did not answer 200 (not probed when +// --public-url is empty: every run then logs a warning) +// +// The counters of the status count from the server's start, so a restart +// starts them at zero: after one, whatever they count is new. The first run +// (no state file) takes the counters as they are and says nothing about them. +// +// An alert about something that happens (degraded-search, breaker-open, +// provider-rejected, consolidation-dropped, link-fallback, add-outran and +// restart: the others are about how things are) is firing for 10 minutes +// after the last run that saw it happen, so that a stream of events is one +// alert, repeated every 30 minutes while it lasts and then resolved, not a +// pair of messages per event. +// Any firing alert is said again 30 minutes after its last message. When the +// status cannot be read, the alerts that come from it are neither repeated +// nor resolved: the ready alert says what is wrong. + +const ( + // watchRepeat is how long after its last message a firing alert is said + // again. + watchRepeat = 30 * time.Minute + // watchHold is how long an alert about a counter keeps firing after the + // last time the counter was seen to rise. + watchHold = 10 * time.Minute + + watchBacklogAge = 15 * time.Minute + watchLagFor = 15 * time.Minute + // watchBudgetUsed is the share of a cap spent and held above which an + // account alerts; watchBudgetHeld, the share held alone. + watchBudgetUsed = 0.80 + watchBudgetHeld = 0.02 + // watchPartialPercent is the share of the last 10 minutes' Searches that + // may be partially degraded, in percent, and watchPartialSearches the + // fewest Searches in that window from which the share counts. + watchPartialPercent = 5 + watchPartialSearches = 20 + + // watchMessageBytes bounds a message: under 1,000 characters, and under + // WeCom's 2,048 bytes and Telegram's 4,096 characters whatever the + // characters. + watchMessageBytes = 999 + + // watchTag starts every message, of every kind. A Feishu bot protected by + // a keyword takes only the texts that contain it and answers the others + // 200 with a code, which is a failed delivery; the runbook has the + // operator register the keyword mnemon-aml, so no text may lack it. + watchTag = "[mnemon-aml]" + + watchReadTimeout = 5 * time.Second + watchHookTimeout = 10 * time.Second + // watchRunLimit bounds a run: what it has not sent by then is left to + // the next run. + watchRunLimit = time.Minute + + watchStateVersion = 1 + // watchValueBytes bounds the value an alert keeps in the state. + watchValueBytes = 1000 + watchReadLimit = 1 << 20 + watchAnswerLimit = 64 << 10 +) + +// newAMLWatchCommand is `mnemon memory-agent aml-watch`: one run of the alert +// watcher (the comment above). +func newAMLWatchCommand() *cobra.Command { + command := &cobra.Command{ + Use: "aml-watch", + Short: "Check a running aml-serve once and post its alerts to a Telegram, Feishu or WeCom bot", + Long: `Check a running aml-serve once: read its admin listener's /ready and /admin/status and its +public /health, apply the alert rules, and post a text message for each alert that fires, each +firing alert that is due again (every 30 minutes) and each alert that clears, to the webhook named +by the environment variable AML_ALERT_WEBHOOK, in the payload of AML_ALERT_KIND (telegram, feishu or +wecom); for telegram the webhook is https://api.telegram.org/bot/sendMessage?chat_id=. +Every text starts with [mnemon-aml], the keyword to register on a Feishu bot, and says which host +it is from (and which instance, with --instance). Run it every minute with a systemd timer. The +webhook URL and the admin token are secrets and are never printed. With no webhook the alerts are +logged only. The state between runs is the file --state; one run at a time holds .lock.`, + Args: cobra.NoArgs, + RunE: runAMLWatch, + } + flags := command.Flags() + flags.String("admin", "http://127.0.0.1:8091", "URL of aml-serve's admin listener (--admin-listen)") + flags.String("public-url", "", "URL of the public listener as clients reach it, such as https://memory.example.com, whose /health is checked; empty to skip that check (every run then logs a warning)") + flags.String("state", "", "Absolute path of the state file kept between runs, such as /var/lib/mnemon-aml/watch-state.json (required)") + flags.String("admin-token-env", "", "Environment variable holding the token aml-serve's admin listener requires (its --admin-token-env), sent as a bearer token on every admin read; empty for none") + flags.String("instance", "", "Name of the aml-serve instance this watcher watches, such as beta-1 (2 to 31 lowercase letters, digits and hyphens, starting with a letter): every message then says it is on host/name; empty for the host alone") + return command +} + +// watchSettings are aml-watch's checked settings. webhook is empty to log the +// alerts instead of posting them. adminToken is the bearer token of the admin +// reads, "" for none; instance names the instance in the messages, "" for the +// host alone. +type watchSettings struct { + admin, public, state string + webhook, kind string + adminToken, instance string +} + +// watchInstance is the shape of an instance's name (--instance). +var watchInstance = regexp.MustCompile(`^[a-z][a-z0-9-]{1,30}$`) + +// amlWatchSettings reads and checks aml-watch's flags and environment; it +// touches nothing. The webhook and its kind are checked when the watcher runs. +func amlWatchSettings(cmd *cobra.Command) (watchSettings, error) { + flags := cmd.Flags() + var s watchSettings + admin, _ := flags.GetString("admin") + public, _ := flags.GetString("public-url") + s.state, _ = flags.GetString("state") + s.instance, _ = flags.GetString("instance") + tokenEnv, _ := flags.GetString("admin-token-env") + var err error + if s.admin, err = watchBaseURL("--admin", admin); err != nil { + return s, err + } + if public != "" { + if s.public, err = watchBaseURL("--public-url", public); err != nil { + return s, err + } + } + switch { + case s.state == "": + return s, errors.New("--state is required: the file the watcher keeps between runs") + case !filepath.IsAbs(s.state): + return s, errors.New("--state must be an absolute path") + case s.instance != "" && !watchInstance.MatchString(s.instance): + return s, fmt.Errorf("--instance must be 2 to 31 characters: lowercase letters, digits and hyphens, starting with a letter; not %q", s.instance) + } + if tokenEnv != "" { + // Read once, as it is. aml-serve, which has a rule for the token's + // length, refuses the one that is too short; here it must be there. + if s.adminToken = os.Getenv(tokenEnv); s.adminToken == "" { + return s, fmt.Errorf("--admin-token-env names $%s, which is unset or empty: it must hold the admin token", tokenEnv) + } + } + s.webhook = strings.TrimSpace(os.Getenv("AML_ALERT_WEBHOOK")) + s.kind = strings.ToLower(strings.TrimSpace(os.Getenv("AML_ALERT_KIND"))) + return s, nil +} + +// watchBaseURL is raw as the base of the requests of flag: an http or https +// URL without credentials, query or trailing slash. +func watchBaseURL(flag, raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" || u.Scheme != "http" && u.Scheme != "https" { + return "", fmt.Errorf("%s must be an http or https URL, such as http://127.0.0.1:8091", flag) + } + if u.User != nil { + return "", fmt.Errorf("%s must not contain credentials", flag) + } + u.RawQuery, u.Fragment = "", "" + return strings.TrimRight(u.String(), "/"), nil +} + +// runAMLWatch is one run: the process's SIGINT and SIGTERM end the requests +// in flight, and the state is saved all the same. +func runAMLWatch(cmd *cobra.Command, _ []string) error { + settings, err := amlWatchSettings(cmd) + if err != nil { + return err + } + parent := cmd.Context() + if parent == nil { + parent = context.Background() + } + ctx, stop := signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM) + defer stop() + return newWatcher(settings, amlLogger("text", cmd.ErrOrStderr())).run(ctx) +} + +// watcher is one aml-watch run's world: where to look, where to post, the +// clock and the logger. +type watcher struct { + watchSettings + host string + now func() time.Time + log *slog.Logger + // reads makes the requests to the admin and public listeners, hooks the + // post to the webhook, which is not followed to another address. + reads, hooks *http.Client +} + +func newWatcher(settings watchSettings, log *slog.Logger) *watcher { + host, err := os.Hostname() + if err != nil || host == "" { + host = "unknown-host" + } + return &watcher{ + watchSettings: settings, host: host, now: time.Now, log: log, + reads: &http.Client{Timeout: watchReadTimeout, CheckRedirect: watchReadRedirect}, + hooks: &http.Client{ + Timeout: watchHookTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, + } +} + +// watchReadRedirect is the redirect policy of the reads: net/http's own (ten +// redirects at most), except that a request that carries the admin token goes +// nowhere else, as the Location of an answer is not the operator's. +func watchReadRedirect(req *http.Request, via []*http.Request) error { + if req.Header.Get("Authorization") != "" { + return http.ErrUseLastResponse + } + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + return nil +} + +// origin is what the header of a message says it is on: the host, and with +// --instance, host/name. +func (w *watcher) origin() string { + if w.instance == "" { + return w.host + } + return w.host + "/" + w.instance +} + +// run reads, evaluates, posts and saves, once. Its error says that a message +// could not be delivered, that the state could not be saved or read, or that +// the run was interrupted; an alert, a server that does not answer included, +// is not an error. +func (w *watcher) run(ctx context.Context) error { + target, err := w.checkDelivery() + if err != nil { + return err + } + unlock, otherRun := w.lockRun() + if otherRun { + return nil + } + defer unlock() + ctx, cancel := context.WithTimeout(ctx, watchRunLimit) + defer cancel() + now := w.now().UTC() + prev, err := loadWatchState(w.state, now, w.log) + if err != nil { + return err + } + obs := w.observe(ctx) + if err := ctx.Err(); err != nil { + // An interrupted run saw nothing, whatever its probes made of that: it + // says nothing and keeps the state as it was. + return fmt.Errorf("aml-watch was interrupted before it had read the server: %w", err) + } + next, values, frozen := evaluate(obs, prev, now) + var messages []watchMessage + next.Firing, messages = planAlerts(now, w.origin(), values, frozen, prev.Firing) + outcomes, deliveryErr := w.deliver(ctx, target, messages) + settleAlerts(next.Firing, messages, outcomes, now) + return errors.Join(deliveryErr, saveWatchState(w.state, next)) +} + +// lockRun takes the exclusive, non-blocking lock on .lock for the run, +// so that two runs (the timer's and an operator's, say) do not overlap: they +// would post the same messages twice, and the last to save would win. If +// another run holds it, otherRun is true: the run has said so and does +// nothing (no post, no save) and exits 0. A lock that cannot be opened at all +// (a read-only directory) does not stop the run: delivery matters more than +// the lock, so the run goes on without it, and says so. unlock gives the lock +// back, closing its file. +func (w *watcher) lockRun() (unlock func(), otherRun bool) { + path := w.state + ".lock" + noLock := func() {} + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + w.log.Warn("aml-watch: could not take the run lock; going on without it", "lock", path, "error", err) + return noLock, false + } + lock := flock.New(path) + locked, err := lock.TryLock() + switch { + case err != nil: + w.log.Warn("aml-watch: could not take the run lock; going on without it", "lock", path, "error", err) + return noLock, false + case !locked: + w.log.Warn("aml-watch: another aml-watch run holds the lock; this run does nothing", "lock", path) + return noLock, true + } + return func() { + if err := lock.Unlock(); err != nil { + w.log.Warn("aml-watch: could not give the run lock back", "lock", path, "error", err) + } + }, false +} + +// hookTarget is where a run posts its messages: the webhook's URL, or for a +// Telegram bot its sendMessage without the query, with the chat the payload +// names ("" for the other kinds). url is "" without a webhook. +type hookTarget struct{ url, chat string } + +// checkDelivery checks the webhook and its kind, and is where the run's +// messages go: the webhook is parsed once a run, not once a message (the +// final review's parked T15-h). Without a webhook there is nothing to check, +// and the target is empty. +func (w *watcher) checkDelivery() (hookTarget, error) { + if w.webhook == "" { + return hookTarget{}, nil + } + if w.kind != "feishu" && w.kind != "wecom" && w.kind != "telegram" { + return hookTarget{}, errors.New("AML_ALERT_KIND must be feishu or wecom or telegram") + } + // The errors never repeat the value: its path is a secret. + u, err := url.Parse(w.webhook) + if err != nil || u.Host == "" || !(u.Scheme == "https" || u.Scheme == "http" && watchLoopbackHost(u.Hostname())) { + return hookTarget{}, errors.New("AML_ALERT_WEBHOOK must be an https URL (http only to localhost or a loopback address)") + } + target := hookTarget{url: w.webhook} + if w.kind == "telegram" { + if target.url, target.chat, err = telegramTarget(u); err != nil { + return hookTarget{}, err + } + } + return target, nil +} + +// watchTelegramChat is the shape of a Telegram chat id: a chat's number (a +// group's or a channel's is negative), or a public channel's @username. +var watchTelegramChat = regexp.MustCompile(`^(-?[0-9]{1,20}|@[A-Za-z][A-Za-z0-9_]{4,31})$`) + +// telegramTarget is where a Telegram bot's messages go, from the webhook u, +// which must be the bot's sendMessage with the chat in the query and nothing +// else there: https://api.telegram.org/bot/sendMessage?chat_id=, +// on any host (a self-hosted Bot API server answers the same path). target is +// u without its query, and chat the chat_id as given, which the body carries. +// The errors say the rule, never the value: the path holds the bot's token. +func telegramTarget(u *url.URL) (target, chat string, err error) { + if !strings.HasSuffix(u.Path, "/sendMessage") { + return "", "", errors.New("AML_ALERT_WEBHOOK for telegram must be the bot's sendMessage URL: https://api.telegram.org/bot/sendMessage?chat_id=") + } + query, err := url.ParseQuery(u.RawQuery) + if chats := query["chat_id"]; err != nil || len(query) != 1 || len(chats) != 1 || !watchTelegramChat.MatchString(chats[0]) { + return "", "", errors.New("AML_ALERT_WEBHOOK for telegram must end in ?chat_id=, its only parameter: a chat's number (negative for a group or a channel) or a channel's @username") + } + bare := *u + bare.RawQuery, bare.ForceQuery, bare.Fragment, bare.RawFragment = "", false, "", "" + return bare.String(), query.Get("chat_id"), nil +} + +// watchLoopbackHost reports whether host is this machine: localhost, in any +// case, or a loopback IP address. +func watchLoopbackHost(host string) bool { + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// ---- what the run reads ---- + +// watchObservation is what one run read. status is nil when /admin/status +// could not be read; each other field is "" when its probe was fine, else +// says what was wrong. +type watchObservation struct { + status *aml.AdminStatus + statusErr string + ready string + health string +} + +// observe reads /ready, /admin/status and, with a public URL, /health. +func (w *watcher) observe(ctx context.Context) watchObservation { + var obs watchObservation + obs.ready = w.probeReady(ctx) + obs.status, obs.statusErr = w.readStatus(ctx) + if w.public != "" { + obs.health = w.probeHealth(ctx) + } else { + // The probe through the public URL is the only check of the path + // clients take (DNS, TLS, the proxy): a forgotten flag leaves a trace. + w.log.Warn("aml-watch: the public health probe is off because --public-url is empty") + } + return obs +} + +// fetch GETs base+path and returns the status and the start of the body. A +// token, when not empty, is sent as the request's bearer token: the admin +// reads pass the admin token, the public one none. +func (w *watcher) fetch(ctx context.Context, base, path, token string) (int, []byte, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(base, "/")+path, nil) + if err != nil { + return 0, nil, err + } + request.Header.Set("Accept", "application/json") + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := w.reads.Do(request) + if err != nil { + return 0, nil, err + } + defer response.Body.Close() + body, err := io.ReadAll(io.LimitReader(response.Body, watchReadLimit)) + return response.StatusCode, body, err +} + +// watchRequestError is err without the URL net/http puts in front of it. +func watchRequestError(err error) string { + var urlErr *url.Error + if errors.As(err, &urlErr) { + err = urlErr.Err + } + return err.Error() +} + +func (w *watcher) probeReady(ctx context.Context) string { + code, body, err := w.fetch(ctx, w.admin, "/ready", w.adminToken) + switch { + case err != nil: + return "/ready did not answer: " + watchRequestError(err) + case code == http.StatusOK: + return "" + } + var ready struct{ Reasons []string } + if code == http.StatusServiceUnavailable && json.Unmarshal(body, &ready) == nil && len(ready.Reasons) > 0 { + return "/ready answered 503: " + strings.Join(ready.Reasons, "; ") + } + return fmt.Sprintf("/ready answered %d", code) +} + +func (w *watcher) readStatus(ctx context.Context) (*aml.AdminStatus, string) { + code, body, err := w.fetch(ctx, w.admin, "/admin/status", w.adminToken) + if err != nil { + return nil, "/admin/status did not answer: " + watchRequestError(err) + } + if code != http.StatusOK { + return nil, fmt.Sprintf("/admin/status answered %d", code) + } + var status aml.AdminStatus + if json.Unmarshal(body, &status) != nil || status.StartedAt.IsZero() { + return nil, "/admin/status did not answer with an aml-serve status" + } + return &status, "" +} + +func (w *watcher) probeHealth(ctx context.Context) string { + target := strings.TrimRight(w.public, "/") + "/health" + code, _, err := w.fetch(ctx, w.public, "/health", "") + switch { + case err != nil: + return fmt.Sprintf("GET %s did not answer: %s", target, watchRequestError(err)) + case code != http.StatusOK: + return fmt.Sprintf("GET %s answered %d", target, code) + } + return "" +} + +// ---- the state between runs ---- + +// watchState is the state file: JSON, rewritten whole by every run. +// +// startedAt the start of the server whose counters these are (zero before +// any status was read) +// counters the status's counters as of the last run that read it, by name +// (the status's own for the Search and consolidation counters, +// "trips:" for a kind's breaker trips, "rejected:" for +// the requests it rejected) +// rises when each alert about a counter was last seen to be new +// since when each condition that must last (backlog-lag) was first seen +// firing the alerts that fired and have not been resolved +type watchState struct { + Version int `json:"version"` + CheckedAt time.Time `json:"checkedAt"` + StartedAt time.Time `json:"startedAt"` + Counters map[string]int64 `json:"counters"` + Rises map[string]time.Time `json:"rises"` + Since map[string]time.Time `json:"since"` + Firing map[string]watchAlert `json:"firing"` +} + +// watchAlert is an alert in the state: since when it fires, when its message +// last went out (zero if none did: it is sent by the next run), and the value +// that fired it last. An alert that stopped firing stays in the state until +// its resolution is delivered. +type watchAlert struct { + Since time.Time `json:"since"` + LastSent time.Time `json:"lastSent"` + Value string `json:"value"` +} + +func newWatchState() watchState { + return watchState{Version: watchStateVersion, Counters: map[string]int64{}, Rises: map[string]time.Time{}, Since: map[string]time.Time{}, Firing: map[string]watchAlert{}} +} + +// loadWatchState reads the state file. A missing file is a first run. A file +// that is not a state of this version is one too, said in the log, but it is +// moved aside as .bad-