From 4c9e9e0f446c9b872be6c0eb80b121e461180e32 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 17 Sep 2026 16:39:19 -0700 Subject: [PATCH 1/7] Initial EC2 changes --- README.md | 2 + canyonos_core/cli.py | 1 - .../cloud_provider_logic/EC2/README.md | 125 +++++++++++++----- .../cloud_provider_logic/EC2/_runtime.py | 58 ++++++-- .../cloud_provider_logic/Local/README.md | 35 +++++ .../controller/cloud_provider_logic/README.md | 13 ++ tests/test_runtime_ec2.py | 71 +++++++++- 7 files changed, 256 insertions(+), 49 deletions(-) create mode 100644 canyonos_core/controller/cloud_provider_logic/Local/README.md create mode 100644 canyonos_core/controller/cloud_provider_logic/README.md diff --git a/README.md b/README.md index d097c241..9a3ff693 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,8 @@ env_file: .env If you are deploying agents and tools to multiple hosts, make sure the hosts are reachable from the machine running the deploy command and that SSH key-based access is already configured. A guide to set that up can be found [here](https://www.redhat.com/en/blog/passwordless-ssh). +For `provider: EC2` agents specifically (AMI/IAM/security group requirements, the `ec2:` config block), see [canyonos_core/controller/cloud_provider_logic](canyonos_core/controller/cloud_provider_logic/README.md). + ### 6. Stop or quit ```bash diff --git a/canyonos_core/cli.py b/canyonos_core/cli.py index 2064c365..e83aa7fd 100644 --- a/canyonos_core/cli.py +++ b/canyonos_core/cli.py @@ -24,7 +24,6 @@ ARTIFACT_DIR_NAME = ".car" SOURCE_DIR_NAME = "app" EC2_REQUIRED_CONFIG_KEYS = ( - "ami_id", "subnet_id", "security_group_ids", "region", diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/README.md b/canyonos_core/controller/cloud_provider_logic/EC2/README.md index ea9ee2b1..19408c79 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/README.md +++ b/canyonos_core/controller/cloud_provider_logic/EC2/README.md @@ -1,48 +1,111 @@ -What you need to run agents on EC2. +# EC2 Specific Set-up -For local controller -- AMI with the following installed: - - Docker - - Public key in authorizedkeys in ~/.ssh +The backend for `provider: EC2` agents. It launches one EC2 instance per +replica, ships the agent's built Docker image to it over SSH, and starts the +container there. -For global controller -- Instance with all of the following installed: - - Both things in local controller - - Python 3.10+ - - CanyonOS folder - - pip requirements installed in env - - pip install -e . --break-system-packages - - Private key labeled as ventis_ec2 inside ~/.ssh - - Private key complementing local public key +## What the host needs +** The host means the machine that you are running canyonos deploy on ** -For convenience, you can use the global controller as the -AMI for local to save some time +- Docker running locally — the agent image is built and `docker save`d here before transfer. +- `zstd` on `$PATH` — the image is piped through it before the SSH transfer. +- AWS Credentials. Would need the IAM permissions to execute certain EC2 commands on the machine. The ec2_launcher role in IAM covers all of the needed permissions. For exact knowledge, look below in the `IAM Permissions` section. -AWS Specific Permissions -- IAM Role to attach to global controller instance allowing - - ec2:RunInstance,TerminateInstance,DescribeInstances,CreateTags -- SSH permissions for entering global controller -- Security group allowing port 50051, 6379, and 22 TCP communication within security group -- Also need the port your workflow API's are open to be allowed +## AWS-side setup +- **IAM instance profile** named exactly `ec2launch` — hardcoded in + `provision_instance`'s `run_instances` call, so the account must have a + profile by this literal name. Its role should grant whatever the agent + code itself needs on AWS (e.g. Bedrock, if not going through the LLM proxy). +- **Security group** (`ec2.security_group_ids`) allowing inbound TCP `22` (SSH), + `50051` (agent gRPC / health check), and each agent's `redis_port` (default + `6379`) from the deploying host. A `type: workflow` agent's `api_port` + (default `8080`) needs inbound access too if you're calling it from off-box. -Steps: +## Config (`ec2:` block in `global_controller.yaml`) -1. SSH into EC2 Global Controller Instance +You need to add this block to global_controller.yaml, to give the deploy the necessary credentials to launch agents on EC2. -2. Create your app +```yaml +ec2: + region: us-east-1 + subnet_id: subnet-0123456789abcdef0 + security_group_ids: + - sg-0123456789abcdef0 +``` -3. Change the agents/configs/workflow folders to suit your needs +Values also accept `${VAR}` interpolation from `env_file` (see `examples/portfolio`). -4. Run canyonos build + canyonos deploy -For cleanup, use canyonos clean to clean stubs/containers +`canyonos build` handles this automatically, but each EC2 agent's spec also needs its own `instance_type` (e.g. `t3.micro`), example below. -If encountering permission errors with the keys, run this to give key more permissions if blocked -chmod 700 ~/.ssh -chmod 600 ~/.ssh/ventis_ec2 +Example: +```yaml + - name: ExampleAgent + entrypoint: agents/example_agent.py + provider: EC2 + instance_type: t3.micro +``` +## IAM permissions +The host machine needs these IAM Permissions to be able to launch external EC2 instances. This can either be held by the host itself, or the EC2 machine hosting the deployment. +- ec2: + - RunInstances + - TerminateInstances + - DescribeInstances + - CreateTags + - ImportKeyPair +- IAM: + - PassRole + +You can paste the exact JSON below in the "create manual policy" field in IAM when generating permissions for a role + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:RunInstances", + "ec2:TerminateInstances", + "ec2:DescribeInstances", + "ec2:CreateTags", + "ec2:ImportKeyPair" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "arn:aws:iam:::role/" + } + ] +} +``` + +## [OPTIONAL] Private Key + +By default, canyonos creates a new private key for every project you deploy (The key pair is named `canyonos-ec2--`), but you can set your own key pair in the global_controller.yaml file in the ec2 block. +In addition to the security group and subnet_id, if you place `ssh_private_key_path: path/to/private/key` there, the key used will instead be that. + +## [OPTIONAL] Creating your own AMI + +We have provided our own base AMI_ID: ami-0101d5f2a2a9cd55c, but if you want to create your own, the ami you create just needs to have docker and zstd installed. The commands to install it are below. + +```bash + +#!/bin/bash +set -eux + +apt-get update +apt-get install -y docker.io zstd + +systemctl enable docker +systemctl start docker + +``` \ No newline at end of file diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py b/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py index 058f2ec6..41560f95 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py +++ b/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py @@ -12,6 +12,7 @@ they can read config and reuse the controller's Docker/Redis logic. """ +import hashlib import logging import os import shlex @@ -22,6 +23,7 @@ from typing import Any import boto3 +from botocore.exceptions import ClientError from canyonos_core.controller.utils.container_names import container_name from canyonos_core.controller.utils.env_file import env_file_args @@ -32,7 +34,13 @@ CONTAINER_PORT = 50051 PROVIDER = "ec2" +PUBLIC_IP_TIMEOUT = 120 +CONTROLLER_HEALTH_TIMEOUT = 180 DEFAULT_SSH_KEY_PATH = os.path.expanduser("~/.ssh/ventis_ec2") +DEFAULT_SSH_USER = "ubuntu" + +# This default AMI is a public AMI created by Canyon Code, containing base Ubuntu + Docker + zstd. Can be overriden manually with your own ami_id +DEFAULT_AMI_ID = "ami-0101d5f2a2a9cd55c" _controller: Any = None @@ -59,18 +67,45 @@ def _ssh_key_path(cfg): def _aws_clients(): """Return validated EC2 config and EC2 client.""" cfg = _controller.config.get("ec2", {}) + cfg.setdefault("ami_id", DEFAULT_AMI_ID) + cfg.setdefault("ssh_user", DEFAULT_SSH_USER) required = [ - "ami_id", "subnet_id", "security_group_ids", "region", - "ssh_user", ] missing = [field for field in required if not cfg.get(field)] if missing: raise ValueError(f"Missing EC2 config: {', '.join(sorted(missing))}") - _ssh_key_path(cfg) - return cfg, boto3.client("ec2", region_name=cfg["region"]) + key_path = _ssh_key_path(cfg) + client = boto3.client("ec2", region_name=cfg["region"]) + _ensure_key_pair_imported(cfg, key_path, client) + return cfg, client + + +def _ensure_key_pair_imported(cfg, key_path, client): + """Import the controller's SSH public key as a per-project EC2 key pair, if it isn't already. + + Naming the key pair after the project id and a hash of the public key itself + (rather than one fixed name) keeps unrelated projects, and a key that gets + regenerated after the original file is lost, from colliding under the same + AWS key pair name and silently drifting out of sync with it. + """ + public_key = subprocess.run( + ["ssh-keygen", "-y", "-f", key_path], + capture_output=True, + text=True, + check=True, + ).stdout.encode() + project_id = _controller.config.get("project_id") or "default" + fingerprint = hashlib.sha256(public_key).hexdigest()[:8] + key_name = f"canyonos-ec2-{project_id}-{fingerprint}" + try: + client.import_key_pair(KeyName=key_name, PublicKeyMaterial=public_key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") != "InvalidKeyPair.Duplicate": + raise + cfg["key_pair_name"] = key_name def provision_instance(spec, replica_index, next_host_port=None): @@ -82,6 +117,7 @@ def provision_instance(spec, replica_index, next_host_port=None): "InstanceType": spec["instance_type"], "SubnetId": cfg["subnet_id"], "SecurityGroupIds": cfg["security_group_ids"], + "KeyName": cfg["key_pair_name"], "MinCount": 1, "MaxCount": 1, "IamInstanceProfile": {"Name": "ec2launch"}, @@ -110,7 +146,7 @@ def provision_instance(spec, replica_index, next_host_port=None): runtime_id = f"{container_name(agent_name, replica_index)}--{instance_id}" client.get_waiter("instance_running").wait(InstanceIds=[instance_id]) - deadline = time.time() + cfg.get("public_ip_timeout", 120) + deadline = time.time() + PUBLIC_IP_TIMEOUT instance = None while time.time() < deadline: response = client.describe_instances(InstanceIds=[instance_id]) @@ -171,10 +207,7 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): redis_port=redis_port, agent_id=agent_id, ) - _check_controller_health( - f"{host}:{CONTAINER_PORT}", - timeout=cfg.get("controller_health_timeout", 180), - ) + _check_controller_health(f"{host}:{CONTAINER_PORT}") instance = { "agent_name": spec["name"], "provider": "EC2", @@ -321,13 +354,10 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, ) -def _check_controller_health(endpoint, timeout=None): +def _check_controller_health(endpoint, timeout=CONTROLLER_HEALTH_TIMEOUT): """Wait until the launched container accepts TCP connections.""" host, port = endpoint.split(":") - deadline = time.time() + ( - timeout - or _controller.config.get("ec2", {}).get("controller_health_timeout", 180) - ) + deadline = time.time() + timeout while time.time() < deadline: try: with socket.create_connection((host, int(port)), timeout=2): diff --git a/canyonos_core/controller/cloud_provider_logic/Local/README.md b/canyonos_core/controller/cloud_provider_logic/Local/README.md new file mode 100644 index 00000000..feefe547 --- /dev/null +++ b/canyonos_core/controller/cloud_provider_logic/Local/README.md @@ -0,0 +1,35 @@ +# Local Set-up + +The default backend for `provider: local` agents. Runs each agent as a Docker +container, normally on the same machine you run `canyonos deploy` on. + +## What you need + +- Docker running locally. That's it — the `canyonos-local` network and each + host's Redis container are created for you automatically. + +## Remote hosts (optional) + +You're not limited to your own machine — point an agent at another box you +already have running by setting `host` (and `user`, if needed) on that agent: + +```yaml + - name: MetricsAgent + provider: local + host: 10.0.0.12 + user: ubuntu +``` + +That machine needs: +- Docker running, reachable without a sudo password prompt over SSH. +- Passwordless SSH access from the machine you deploy from, using the key at + `ec2.ssh_private_key_path` (default `~/.ssh/ventis_ec2`) — set this even if + you have no EC2 agents, since remote `local` hosts use the same key. + +## Config + +| Key | Where | Default | Notes | +| --- | --- | --- | --- | +| `host` | per agent | `localhost` | Point this at a remote machine to run that agent there instead. | +| `host_port` / `port` | per agent | next free port from `8000` | Port on the host that maps to the container. | +| `user` | per agent | — | SSH user for a remote `host`. Leave unset for `localhost`. | diff --git a/canyonos_core/controller/cloud_provider_logic/README.md b/canyonos_core/controller/cloud_provider_logic/README.md new file mode 100644 index 00000000..8abdad6a --- /dev/null +++ b/canyonos_core/controller/cloud_provider_logic/README.md @@ -0,0 +1,13 @@ +# Cloud Provider Logic + +Backend implementations for where CanyonOS runs an agent's container. Every agent +in `global_controller.yaml` picks one via `provider: local` or `provider: EC2`. + +What each provider differs in is where the launched agent lives in, local lives in the host terminal and EC2 would spawn in another EC2 instance. + +## Providers + +| Provider | Folder | Compute | +| --- | --- | --- | +| `local` (default) | `Local/` | Docker container on the same machine, the root README has the whole flow. | +| `EC2` | `EC2/` | One EC2 instance per replica, needs specific IAM permissions set-up, see [EC2/README.md](EC2/README.md). | diff --git a/tests/test_runtime_ec2.py b/tests/test_runtime_ec2.py index 6120097e..f8c500c1 100644 --- a/tests/test_runtime_ec2.py +++ b/tests/test_runtime_ec2.py @@ -1,10 +1,13 @@ import os +import subprocess import sys import tempfile import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch +from botocore.exceptions import ClientError + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from canyonos_core.controller.cloud_provider_logic.EC2 import _runtime as ec2_runtime @@ -25,8 +28,13 @@ def __init__(self, public_ip="54.10.20.30", private_ip="10.0.0.30"): self.instances = {} self.run_requests = [] self.terminate_requests = [] + self.import_key_pair_requests = [] self.waiter = _FakeWaiter() + def import_key_pair(self, **kwargs): + self.import_key_pair_requests.append(kwargs) + return {} + def run_instances(self, **kwargs): self.run_requests.append(kwargs) instance_id = f"i-test{len(self.run_requests)}" @@ -62,6 +70,11 @@ def setUp(self): key_file = tempfile.NamedTemporaryFile(delete=False) key_file.close() self.key_path = key_file.name + os.unlink(self.key_path) + subprocess.run( + ["ssh-keygen", "-t", "ed25519", "-N", "", "-f", self.key_path, "-q"], + check=True, + ) os.chmod(self.key_path, 0o600) self.fake_client = _FakeEC2Client() self.client_calls = [] @@ -75,7 +88,6 @@ def setUp(self): "region": "us-east-1", "ssh_user": "ubuntu", "ssh_private_key_path": self.key_path, - "public_ip_timeout": 1, }, }, registry_url=None, @@ -94,6 +106,7 @@ def setUp(self): def tearDown(self): self.client_patch.stop() os.unlink(self.key_path) + os.unlink(f"{self.key_path}.pub") ec2_runtime._controller = self.original_controller def _make_client(self, service_name, region_name=None): @@ -104,11 +117,25 @@ def _make_client(self, service_name, region_name=None): return self.fake_client def test_aws_clients_fails_when_required_fields_are_missing(self): - self.controller.config["ec2"].pop("ami_id") + self.controller.config["ec2"].pop("subnet_id") with self.assertRaisesRegex(ValueError, "Missing EC2 config"): ec2_runtime._aws_clients() + def test_aws_clients_defaults_ami_id_when_missing(self): + self.controller.config["ec2"].pop("ami_id") + + cfg, _ = ec2_runtime._aws_clients() + + self.assertEqual(cfg["ami_id"], ec2_runtime.DEFAULT_AMI_ID) + + def test_aws_clients_defaults_ssh_user_when_missing(self): + self.controller.config["ec2"].pop("ssh_user") + + cfg, _ = ec2_runtime._aws_clients() + + self.assertEqual(cfg["ssh_user"], ec2_runtime.DEFAULT_SSH_USER) + def test_aws_clients_rejects_missing_ssh_private_key(self): self.controller.config["ec2"]["ssh_private_key_path"] = ( "/tmp/missing-canyonos-key" @@ -123,6 +150,40 @@ def test_aws_clients_rejects_insecure_ssh_private_key_permissions(self): with self.assertRaisesRegex(ValueError, "insecure permissions 0644"): ec2_runtime._aws_clients() + def test_key_pair_name_scoped_by_project_id_and_stable_for_same_key(self): + cfg, _ = ec2_runtime._aws_clients() + first_name = cfg["key_pair_name"] + self.assertRegex(first_name, r"^canyonos-ec2-default-[0-9a-f]{8}$") + + cfg, _ = ec2_runtime._aws_clients() + self.assertEqual(cfg["key_pair_name"], first_name) + + self.controller.config["project_id"] = "my-project" + cfg, _ = ec2_runtime._aws_clients() + self.assertNotEqual(cfg["key_pair_name"], first_name) + self.assertTrue(cfg["key_pair_name"].startswith("canyonos-ec2-my-project-")) + + def test_aws_clients_tolerates_already_imported_key_pair(self): + self.fake_client.import_key_pair = MagicMock( + side_effect=ClientError( + {"Error": {"Code": "InvalidKeyPair.Duplicate", "Message": "boom"}}, + "ImportKeyPair", + ) + ) + + ec2_runtime._aws_clients() + + def test_aws_clients_reraises_other_key_pair_errors(self): + self.fake_client.import_key_pair = MagicMock( + side_effect=ClientError( + {"Error": {"Code": "UnauthorizedOperation", "Message": "boom"}}, + "ImportKeyPair", + ) + ) + + with self.assertRaises(ClientError): + ec2_runtime._aws_clients() + def test_provision_uses_ec2_client(self): spec = { "name": "Tagged", @@ -135,8 +196,12 @@ def test_provision_uses_ec2_client(self): request = self.fake_client.run_requests[0] self.assertEqual(request["ImageId"], "ami-123456") - self.assertNotIn("KeyName", request) + self.assertRegex(request["KeyName"], r"^canyonos-ec2-default-[0-9a-f]{8}$") self.assertNotIn("UserData", request) + self.assertEqual( + self.fake_client.import_key_pair_requests[0]["KeyName"], + request["KeyName"], + ) self.assertEqual( request["TagSpecifications"][0]["Tags"][0], {"Key": "Name", "Value": "canyonos-Tagged-2"}, From 2e438767e3a289246b34b1ee430cc3414cbe6590 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 17 Sep 2026 16:54:17 -0700 Subject: [PATCH 2/7] Draft 2 --- .../cloud_provider_logic/EC2/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/README.md b/canyonos_core/controller/cloud_provider_logic/EC2/README.md index 19408c79..58539e49 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/README.md +++ b/canyonos_core/controller/cloud_provider_logic/EC2/README.md @@ -4,13 +4,13 @@ The backend for `provider: EC2` agents. It launches one EC2 instance per replica, ships the agent's built Docker image to it over SSH, and starts the container there. -## What the host needs +## What the host needs to deploy to EC2 ** The host means the machine that you are running canyonos deploy on ** -- Docker running locally — the agent image is built and `docker save`d here before transfer. -- `zstd` on `$PATH` — the image is piped through it before the SSH transfer. -- AWS Credentials. Would need the IAM permissions to execute certain EC2 commands on the machine. The ec2_launcher role in IAM covers all of the needed permissions. For exact knowledge, look below in the `IAM Permissions` section. +- Everything needed for local deployment. (This is Docker and the canyonos CLI) +- AWS Credentials. The host needs IAM permissions to execute certain EC2 commands. The `AmazonEC2FullAccess` policy in IAM covers all of the needed permissions. For the exact IAM permissions needed, look below in the `IAM Permissions` section. +- Config block added to `global_controller.yaml` (shown below). The security group would also need to allow specific ports in. ## AWS-side setup @@ -18,14 +18,14 @@ container there. `provision_instance`'s `run_instances` call, so the account must have a profile by this literal name. Its role should grant whatever the agent code itself needs on AWS (e.g. Bedrock, if not going through the LLM proxy). -- **Security group** (`ec2.security_group_ids`) allowing inbound TCP `22` (SSH), - `50051` (agent gRPC / health check), and each agent's `redis_port` (default - `6379`) from the deploying host. A `type: workflow` agent's `api_port` - (default `8080`) needs inbound access too if you're calling it from off-box. +- **Security group** (everything is inbound, same permissions needed for outbound unless you allow all outbound traffic) + - The workflow and dashboard port need to be opened up, they are by default 8080 and 8081, and are referred as such below + - Port 50051, 6379, 22 needs to be accessible by the security group + - Port 8080, 8081 needs be accessible by whoever queries the workflow (0.0.0.0 for example) ## Config (`ec2:` block in `global_controller.yaml`) -You need to add this block to global_controller.yaml, to give the deploy the necessary credentials to launch agents on EC2. +You need to add this block to global_controller.yaml, to give canyonos the necessary credentials to launch agents on EC2. ```yaml ec2: From da0f2a825a1e7916b8b995648e5b3758dbb14693 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 17 Sep 2026 17:43:25 -0700 Subject: [PATCH 3/7] Draft 3 --- .../cloud_provider_logic/EC2/README.md | 49 +++++++++++---- .../cloud_provider_logic/EC2/_runtime.py | 3 +- .../cloud_provider_logic/Local/README.md | 24 +------- .../cloud_provider_logic/Local/_runtime.py | 42 +++++++------ canyonos_core/controller/global_controller.py | 22 ++++--- canyonos_core/controller/instance_manager.py | 7 ++- tests/test_global_controller_cleanup.py | 17 ++++++ tests/test_global_controller_redis_reuse.py | 59 +++++++++++++++++-- tests/test_instance_manager_runtime.py | 25 ++++++++ tests/test_runtime_ec2.py | 12 ++++ 10 files changed, 188 insertions(+), 72 deletions(-) diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/README.md b/canyonos_core/controller/cloud_provider_logic/EC2/README.md index 58539e49..500e1755 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/README.md +++ b/canyonos_core/controller/cloud_provider_logic/EC2/README.md @@ -14,14 +14,10 @@ container there. ## AWS-side setup -- **IAM instance profile** named exactly `ec2launch` — hardcoded in - `provision_instance`'s `run_instances` call, so the account must have a - profile by this literal name. Its role should grant whatever the agent - code itself needs on AWS (e.g. Bedrock, if not going through the LLM proxy). - **Security group** (everything is inbound, same permissions needed for outbound unless you allow all outbound traffic) - The workflow and dashboard port need to be opened up, they are by default 8080 and 8081, and are referred as such below - Port 50051, 6379, 22 needs to be accessible by the security group - - Port 8080, 8081 needs be accessible by whoever queries the workflow (0.0.0.0 for example) + - Port 8080 needs be accessible by whoever queries the workflow (0.0.0.0 for example) ## Config (`ec2:` block in `global_controller.yaml`) @@ -35,9 +31,6 @@ ec2: - sg-0123456789abcdef0 ``` -Values also accept `${VAR}` interpolation from `env_file` (see `examples/portfolio`). - - `canyonos build` handles this automatically, but each EC2 agent's spec also needs its own `instance_type` (e.g. `t3.micro`), example below. Example: @@ -60,9 +53,9 @@ The host machine needs these IAM Permissions to be able to launch external EC2 i - CreateTags - ImportKeyPair - IAM: - - PassRole + - PassRole (only needed if you set `ec2.instance_profile_name`) -You can paste the exact JSON below in the "create manual policy" field in IAM when generating permissions for a role +You can paste the exact JSON below in the "create manual policy" field in IAM when generating permissions for a role. Drop the `iam:PassRole` statement if you're not setting `ec2.instance_profile_name`. ```json { @@ -78,16 +71,47 @@ You can paste the exact JSON below in the "create manual policy" field in IAM wh "ec2:ImportKeyPair" ], "Resource": "*" - }, + } + ] +} +``` + + +## [OPTIONAL] Extra Configurations in the ec2 block in `global_controller.yaml` + +The ec2 config block given above has the bare minimum needed to get the instances up and running, to add separate settings, you can add these following fields in the block. + +```yaml +ec2: + # Required Commands + region: us-east-1 + subnet_id: subnet-0123456789abcdef0 + security_group_ids: + - sg-0123456789abcdef0 + + # Optional Commands + instance_profile_name: ec2launch # For the launched instances, if you want a specific IAM role attached to each instance, pass this variable in. + ami_id: ami-0123456789abcdef0 # Look at `Creating your own AMI` below + ssh_user: ubuntu # Look at `Creating your own AMI` below + ssh_private_key_path: ~/.ssh/your-own-key # Look at `Private Key` below +``` +For the instance_profile_name, you will also need to add a new policy to your IAM role in the host machine, the iam:PassRole policy. JSON below. + +```json +{ + "Version": "2012-10-17", + "Statement": [ { "Effect": "Allow", "Action": "iam:PassRole", - "Resource": "arn:aws:iam:::role/" + "Resource": "arn:aws:iam:::role/" } ] } ``` + + ## [OPTIONAL] Private Key By default, canyonos creates a new private key for every project you deploy (The key pair is named `canyonos-ec2--`), but you can set your own key pair in the global_controller.yaml file in the ec2 block. @@ -96,6 +120,7 @@ In addition to the security group and subnet_id, if you place `ssh_private_key_p ## [OPTIONAL] Creating your own AMI We have provided our own base AMI_ID: ami-0101d5f2a2a9cd55c, but if you want to create your own, the ami you create just needs to have docker and zstd installed. The commands to install it are below. +If using a different AMI base than Ubuntu, you will need to change the ssh_user manually. ```bash diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py b/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py index 41560f95..5f98b68b 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py +++ b/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py @@ -120,7 +120,6 @@ def provision_instance(spec, replica_index, next_host_port=None): "KeyName": cfg["key_pair_name"], "MinCount": 1, "MaxCount": 1, - "IamInstanceProfile": {"Name": "ec2launch"}, "TagSpecifications": [ { "ResourceType": "instance", @@ -140,6 +139,8 @@ def provision_instance(spec, replica_index, next_host_port=None): }, ], } + if cfg.get("instance_profile_name"): + request["IamInstanceProfile"] = {"Name": cfg["instance_profile_name"]} response = client.run_instances(**request) instance_id = response["Instances"][0]["InstanceId"] diff --git a/canyonos_core/controller/cloud_provider_logic/Local/README.md b/canyonos_core/controller/cloud_provider_logic/Local/README.md index feefe547..9679d31a 100644 --- a/canyonos_core/controller/cloud_provider_logic/Local/README.md +++ b/canyonos_core/controller/cloud_provider_logic/Local/README.md @@ -1,35 +1,15 @@ # Local Set-up The default backend for `provider: local` agents. Runs each agent as a Docker -container, normally on the same machine you run `canyonos deploy` on. +container on the same machine where you run `canyonos deploy`. ## What you need - Docker running locally. That's it — the `canyonos-local` network and each - host's Redis container are created for you automatically. - -## Remote hosts (optional) - -You're not limited to your own machine — point an agent at another box you -already have running by setting `host` (and `user`, if needed) on that agent: - -```yaml - - name: MetricsAgent - provider: local - host: 10.0.0.12 - user: ubuntu -``` - -That machine needs: -- Docker running, reachable without a sudo password prompt over SSH. -- Passwordless SSH access from the machine you deploy from, using the key at - `ec2.ssh_private_key_path` (default `~/.ssh/ventis_ec2`) — set this even if - you have no EC2 agents, since remote `local` hosts use the same key. + Redis container are created for you automatically. ## Config | Key | Where | Default | Notes | | --- | --- | --- | --- | -| `host` | per agent | `localhost` | Point this at a remote machine to run that agent there instead. | | `host_port` / `port` | per agent | next free port from `8000` | Port on the host that maps to the container. | -| `user` | per agent | — | SSH user for a remote `host`. Leave unset for `localhost`. | diff --git a/canyonos_core/controller/cloud_provider_logic/Local/_runtime.py b/canyonos_core/controller/cloud_provider_logic/Local/_runtime.py index 5d6e0c76..02bac4f3 100644 --- a/canyonos_core/controller/cloud_provider_logic/Local/_runtime.py +++ b/canyonos_core/controller/cloud_provider_logic/Local/_runtime.py @@ -28,26 +28,22 @@ def _require_controller(): return _controller -def _is_local_host(host): - return host in {"localhost", "127.0.0.1"} - - def validate_config(): return None def provision_instance(spec, replica_index, next_host_port): - host = spec.get("host", DEFAULT_HOST) - host_port = int(spec.get("host_port", spec.get("port", next_host_port(host)))) + host_port = int( + spec.get("host_port", spec.get("port", next_host_port(DEFAULT_HOST))) + ) agent_name = spec["name"] return { "provider": PROVIDER, - "host": host, + "host": DEFAULT_HOST, "host_port": host_port, - "redis_host": f"canyonos-redis-{host.replace('.', '-')}", + "redis_host": f"canyonos-redis-{DEFAULT_HOST}", "runtime_id": container_name(agent_name, replica_index), - "user": spec.get("user"), } @@ -56,14 +52,14 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): resources = spec.get("resources", {}) ctrl_type = spec.get("type", "agent") image = f"canyonos-{agent_name.lower()}" - host = provisioned["host"] host_port = provisioned["host_port"] - user = provisioned.get("user") redis_host = provisioned["redis_host"] runtime_id = provisioned["runtime_id"] inspect = _require_controller()._run_cmd( - ["docker", "inspect", "-f", "{{.State.Running}}", runtime_id], host, user + ["docker", "inspect", "-f", "{{.State.Running}}", runtime_id], + DEFAULT_HOST, + None, ) if inspect.returncode == 0 and inspect.stdout.strip() == "true": logger.warning( @@ -71,7 +67,9 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): "treating it as orphaned and recreating.", runtime_id, ) - _require_controller()._run_cmd(["docker", "rm", "-f", runtime_id], host, user) + _require_controller()._run_cmd( + ["docker", "rm", "-f", runtime_id], DEFAULT_HOST, None + ) for attempt in range(MAX_PORT_ATTEMPTS): cmd = [ @@ -131,11 +129,11 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): # User secrets from `env_file`. Explicit -e flags above still win, so a # stray CANYONOS_* line in someone's .env cannot break agent wiring. with env_file_args( - _require_controller(), host, user, runtime_id, _is_local_host(host) + _require_controller(), DEFAULT_HOST, None, runtime_id, True ) as env_args: cmd.extend(env_args) cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) + result = _require_controller()._run_cmd(cmd, DEFAULT_HOST, None) if result.returncode == 0: break @@ -144,7 +142,9 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): # under this name when the port bind fails. Remove it before # retrying with a new port, or the retry hits a name conflict # instead of the port conflict we're trying to work around. - _require_controller()._run_cmd(["docker", "rm", "-f", runtime_id], host, user) + _require_controller()._run_cmd( + ["docker", "rm", "-f", runtime_id], DEFAULT_HOST, None + ) host_port += 1 continue raise RuntimeError(f"Failed to launch {runtime_id}: {result.stderr}") @@ -161,16 +161,14 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): "agent_name": agent_name, "provider": PROVIDER, "replica_index": str(replica_index), - "host": host, + "host": DEFAULT_HOST, "host_port": str(host_port), "container_port": str(CONTAINER_PORT), - "endpoint": f"{host}:{host_port}", + "endpoint": f"{DEFAULT_HOST}:{host_port}", "redis_host": redis_host, "redis_port": str(spec.get("redis_port", 6379)), "runtime_id": runtime_id, } - if user: - instance["user"] = user if ctrl_type == "workflow": instance["api_port"] = str(spec.get("api_port", 8080)) logger.info("Runtime ready: %s -> %s", runtime_id, instance["endpoint"]) @@ -184,8 +182,8 @@ def terminate_instance(instance): result = _require_controller()._run_cmd( ["docker", "rm", "-f", runtime_id], - instance.get("host", DEFAULT_HOST), - instance.get("user"), + DEFAULT_HOST, + None, ) if result.returncode != 0: logger.warning("Failed to remove runtime %s", runtime_id) diff --git a/canyonos_core/controller/global_controller.py b/canyonos_core/controller/global_controller.py index b6193df9..3c44da9c 100644 --- a/canyonos_core/controller/global_controller.py +++ b/canyonos_core/controller/global_controller.py @@ -169,7 +169,8 @@ def _cleanup_stale_containers(self): host_containers = {} for ctrl in self.controllers: - user = ctrl.get("user") + is_ec2 = ctrl.get("provider", "local").upper() == "EC2" + user = ctrl.get("user") if is_ec2 else None placements = self._get_replica_placements(ctrl) for i, (host, port) in enumerate(placements): @@ -277,14 +278,18 @@ def _write_otel_destinations(self, destinations): def _get_replica_placements(ctrl): """Normalize replicas into a list of (host, port) placements.""" replicas = ctrl.get("replicas", 1) - default_host = ctrl.get("host", "localhost") + is_ec2 = ctrl.get("provider", "local").upper() == "EC2" + default_host = ctrl.get("host", "localhost") if is_ec2 else "localhost" base_port = ctrl.get("port", 50051) if isinstance(replicas, int): return [(default_host, base_port + i) for i in range(replicas)] if isinstance(replicas, list): return [ - (r.get("host", default_host), r.get("port", base_port)) + ( + r.get("host", default_host) if is_ec2 else default_host, + r.get("port", base_port), + ) for r in replicas ] return [(default_host, base_port)] @@ -399,7 +404,8 @@ def _launch_redis_containers(self): # Collect unique nodes from all replica placements nodes = {} for ctrl in self.controllers: - user = ctrl.get("user") + is_ec2 = ctrl.get("provider", "local").upper() == "EC2" + user = ctrl.get("user") if is_ec2 else None redis_port = ctrl.get("redis_port", 6379) for host, _port in self._get_replica_placements(ctrl): if host not in nodes: @@ -480,15 +486,13 @@ def _stop_redis_containers(self): for ctrl in self.controllers: if ctrl.get("provider", "local").upper() == "EC2": continue - user = ctrl.get("user") redis_port = ctrl.get("redis_port", 6379) for host, _port in self._get_replica_placements(ctrl): - nodes.setdefault(host, {"user": user, "redis_port": redis_port}) + nodes.setdefault(host, {"redis_port": redis_port}) for host, container_name in self.redis_containers.items(): - user = nodes.get(host, {}).get("user") try: - self._run_cmd(["docker", "stop", container_name], host, user) - self._run_cmd(["docker", "rm", container_name], host, user) + self._run_cmd(["docker", "stop", container_name], host, None) + self._run_cmd(["docker", "rm", container_name], host, None) logger.info("Stopped Redis %s on %s", container_name, host) except Exception as e: logger.warning("Failed to stop Redis %s: %s", container_name, e) diff --git a/canyonos_core/controller/instance_manager.py b/canyonos_core/controller/instance_manager.py index a389d143..83a4be96 100644 --- a/canyonos_core/controller/instance_manager.py +++ b/canyonos_core/controller/instance_manager.py @@ -57,9 +57,12 @@ def ensure_instances(self, agent_specs): reserved_port = None if provider == "local": - host = agent_spec.get("host", local_runtime.DEFAULT_HOST) reserved_port = self._next_host_port( - host, key, agent_name, provider, replica_index + local_runtime.DEFAULT_HOST, + key, + agent_name, + provider, + replica_index, ) jobs.append( diff --git a/tests/test_global_controller_cleanup.py b/tests/test_global_controller_cleanup.py index bce8fb35..2ed88fc6 100644 --- a/tests/test_global_controller_cleanup.py +++ b/tests/test_global_controller_cleanup.py @@ -264,8 +264,10 @@ def _controller(agents, running=False): controller.controllers = agents controller.instance_manager = InstanceManager(controller) controller.removed = [] + controller.command_targets = [] def run_cmd(cmd, host, user=None): + controller.command_targets.append((host, user)) if cmd[:2] == ["docker", "inspect"] and running: return subprocess.CompletedProcess(cmd, 0, "true\n", "") if cmd[:2] == ["docker", "rm"]: @@ -311,6 +313,21 @@ def test_running_replica_is_left_alone(self): self.assertEqual(controller.removed, []) + def test_local_cleanup_ignores_host_and_user_overrides(self): + agents = [ + { + "name": "IntentAgent", + "host": "10.0.0.5", + "user": "ubuntu", + "replicas": [{"host": "10.0.0.6", "port": 50061}], + } + ] + controller = self._controller(agents) + + controller._cleanup_stale_containers() + + self.assertEqual(set(controller.command_targets), {("localhost", None)}) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_global_controller_redis_reuse.py b/tests/test_global_controller_redis_reuse.py index 2f3e74a4..7d254ed8 100644 --- a/tests/test_global_controller_redis_reuse.py +++ b/tests/test_global_controller_redis_reuse.py @@ -73,16 +73,24 @@ def test_an_unhealthy_or_missing_container_still_gets_created(self): ) self.assertIn("localhost", controller.redis_containers) - def test_reuse_check_probes_the_exact_expected_container_name(self): + def test_local_reuse_check_ignores_remote_host_and_user_overrides(self): controller = _bare_controller( - [{"name": "Workflow", "replicas": 1, "redis_port": 6379, "host": "10.0.0.5"}] + [ + { + "name": "Workflow", + "replicas": 1, + "redis_port": 6379, + "host": "10.0.0.5", + "user": "ubuntu", + } + ] ) inspect_calls = [] def fake_run_cmd(cmd, host, user=None): if cmd[:2] == ["docker", "inspect"]: - inspect_calls.append(cmd) + inspect_calls.append((cmd, host, user)) return SimpleNamespace(returncode=0, stdout="true\n", stderr="") return SimpleNamespace(returncode=0, stdout="", stderr="") @@ -95,7 +103,50 @@ def fake_run_cmd(cmd, host, user=None): controller._launch_redis_containers() self.assertEqual(len(inspect_calls), 1) - self.assertIn("canyonos-redis-10-0-0-5", inspect_calls[0]) + cmd, host, user = inspect_calls[0] + self.assertIn("canyonos-redis-localhost", cmd) + self.assertEqual(host, "localhost") + self.assertIsNone(user) + + def test_local_replica_placements_ignore_per_replica_host_overrides(self): + placements = GlobalController._get_replica_placements( + { + "name": "Workflow", + "host": "10.0.0.5", + "replicas": [ + {"host": "10.0.0.6", "port": 50061}, + {"host": "10.0.0.7", "port": 50062}, + ], + } + ) + + self.assertEqual(placements, [("localhost", 50061), ("localhost", 50062)]) + + def test_stop_local_redis_never_uses_ssh_overrides(self): + controller = _bare_controller( + [ + { + "name": "Workflow", + "host": "10.0.0.5", + "user": "ubuntu", + } + ] + ) + controller.redis_containers = {"localhost": "canyonos-redis-localhost"} + calls = [] + controller._run_cmd = lambda cmd, host, user=None: calls.append( + (cmd, host, user) + ) + + controller._stop_redis_containers() + + self.assertEqual( + calls, + [ + (["docker", "stop", "canyonos-redis-localhost"], "localhost", None), + (["docker", "rm", "canyonos-redis-localhost"], "localhost", None), + ], + ) if __name__ == "__main__": diff --git a/tests/test_instance_manager_runtime.py b/tests/test_instance_manager_runtime.py index bb47ddb6..9526a670 100644 --- a/tests/test_instance_manager_runtime.py +++ b/tests/test_instance_manager_runtime.py @@ -185,6 +185,31 @@ def test_local_instances_keep_default_host_and_increment_host_ports(self): ), ) + def test_local_instances_ignore_host_and_user_overrides(self): + controller = _fake_controller() + manager = InstanceManager(controller, controller.redis) + + instance = manager.ensure_instances( + [ + { + "name": "Alpha", + "provider": "local", + "host": "10.0.0.5", + "user": "ubuntu", + } + ] + )[0] + + self.assertEqual(instance["host"], "localhost") + self.assertEqual(instance["endpoint"], "localhost:8000") + self.assertNotIn("user", instance) + self.assertEqual( + controller.redis.hgetall("agent_instance:local:Alpha:0")["host"], + "localhost", + ) + for call in controller._run_cmd.call_args_list: + self.assertEqual(call.args[1:], ("localhost", None)) + def test_bootstrap_instance_passes_poll_interval_env_var(self): controller = _fake_controller() controller.config = {"poll_interval": 7} diff --git a/tests/test_runtime_ec2.py b/tests/test_runtime_ec2.py index f8c500c1..633246c6 100644 --- a/tests/test_runtime_ec2.py +++ b/tests/test_runtime_ec2.py @@ -198,6 +198,7 @@ def test_provision_uses_ec2_client(self): self.assertEqual(request["ImageId"], "ami-123456") self.assertRegex(request["KeyName"], r"^canyonos-ec2-default-[0-9a-f]{8}$") self.assertNotIn("UserData", request) + self.assertNotIn("IamInstanceProfile", request) self.assertEqual( self.fake_client.import_key_pair_requests[0]["KeyName"], request["KeyName"], @@ -213,6 +214,17 @@ def test_provision_uses_ec2_client(self): [{"service_name": "ec2", "region_name": "us-east-1"}], ) + def test_provision_attaches_instance_profile_when_configured(self): + self.controller.config["ec2"]["instance_profile_name"] = "my-custom-profile" + spec = {"name": "Tagged", "provider": "EC2", "instance_type": "t3.small"} + + ec2_runtime.provision_instance(spec, 0) + + request = self.fake_client.run_requests[0] + self.assertEqual( + request["IamInstanceProfile"], {"Name": "my-custom-profile"} + ) + def test_provision_and_bootstrap_instance_return_runtime_record(self): spec = { "name": "Tagged", From c6a870b93c42304fc9aded4015a79853a22b2338 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 17 Sep 2026 17:50:54 -0700 Subject: [PATCH 4/7] Should be final draft --- .../controller/cloud_provider_logic/Local/README.md | 13 ++----------- .../controller/cloud_provider_logic/README.md | 2 +- canyonos_core/controller/global_controller.py | 7 ------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/canyonos_core/controller/cloud_provider_logic/Local/README.md b/canyonos_core/controller/cloud_provider_logic/Local/README.md index 9679d31a..5e6037ee 100644 --- a/canyonos_core/controller/cloud_provider_logic/Local/README.md +++ b/canyonos_core/controller/cloud_provider_logic/Local/README.md @@ -1,15 +1,6 @@ -# Local Set-up +# Local The default backend for `provider: local` agents. Runs each agent as a Docker container on the same machine where you run `canyonos deploy`. -## What you need - -- Docker running locally. That's it — the `canyonos-local` network and each - Redis container are created for you automatically. - -## Config - -| Key | Where | Default | Notes | -| --- | --- | --- | --- | -| `host_port` / `port` | per agent | next free port from `8000` | Port on the host that maps to the container. | +The deployment steps are automatically covered in the root README.md. diff --git a/canyonos_core/controller/cloud_provider_logic/README.md b/canyonos_core/controller/cloud_provider_logic/README.md index 8abdad6a..8d43d03a 100644 --- a/canyonos_core/controller/cloud_provider_logic/README.md +++ b/canyonos_core/controller/cloud_provider_logic/README.md @@ -10,4 +10,4 @@ What each provider differs in is where the launched agent lives in, local lives | Provider | Folder | Compute | | --- | --- | --- | | `local` (default) | `Local/` | Docker container on the same machine, the root README has the whole flow. | -| `EC2` | `EC2/` | One EC2 instance per replica, needs specific IAM permissions set-up, see [EC2/README.md](EC2/README.md). | +| `EC2` | `EC2/` | One EC2 instance per replica. | diff --git a/canyonos_core/controller/global_controller.py b/canyonos_core/controller/global_controller.py index 3c44da9c..72bdc4e5 100644 --- a/canyonos_core/controller/global_controller.py +++ b/canyonos_core/controller/global_controller.py @@ -482,13 +482,6 @@ def _launch_redis_containers(self): def _stop_redis_containers(self): """Stop and remove all launched Redis containers.""" - nodes = {} - for ctrl in self.controllers: - if ctrl.get("provider", "local").upper() == "EC2": - continue - redis_port = ctrl.get("redis_port", 6379) - for host, _port in self._get_replica_placements(ctrl): - nodes.setdefault(host, {"redis_port": redis_port}) for host, container_name in self.redis_containers.items(): try: self._run_cmd(["docker", "stop", container_name], host, None) From 041cc79ea2d3b0314b34954c35eb5c8e43ff4d82 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 17 Sep 2026 17:54:26 -0700 Subject: [PATCH 5/7] Ok final draft fr fr --- canyonos_core/controller/cloud_provider_logic/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/canyonos_core/controller/cloud_provider_logic/README.md b/canyonos_core/controller/cloud_provider_logic/README.md index 8d43d03a..1da06479 100644 --- a/canyonos_core/controller/cloud_provider_logic/README.md +++ b/canyonos_core/controller/cloud_provider_logic/README.md @@ -11,3 +11,5 @@ What each provider differs in is where the launched agent lives in, local lives | --- | --- | --- | | `local` (default) | `Local/` | Docker container on the same machine, the root README has the whole flow. | | `EC2` | `EC2/` | One EC2 instance per replica. | + +See [EC2/README.md](EC2/README.md) for EC2-specific setup. From 15a45b26b5bd56964913d0daf98d23ff5156dd84 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Thu, 17 Sep 2026 19:02:05 -0700 Subject: [PATCH 6/7] Final Rev --- README.md | 4 +- canyonos_core/Dockerfile | 2 +- .../cloud_provider_logic/EC2/README.md | 15 ++++-- .../cloud_provider_logic/EC2/_runtime.py | 35 +++++++++++--- canyonos_core/controller/global_controller.py | 2 +- .../helloworld/config/global_controller.yaml | 2 +- tests/test_runtime_ec2.py | 48 +++++++++++++++++++ 7 files changed, 94 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9a3ff693..77246d26 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ CanyonOS is a control plane that takes your agentic workflow and deploys it, pro Orchestration: Install Kubernetes/Docker Compose for distributed deployment and management -Observability: Install Langfuse/Arize Phoenix for LLM Observability +Observability: Install Langfuse/Arize Phoenix for LLM Observability and then Prometheus/Mimir for Instance Metrics -Execution: Install Ray or Kuberay to manage async task execution +Execution: Install Ray or Kuberay to manage task execution diff --git a/canyonos_core/Dockerfile b/canyonos_core/Dockerfile index cd1c3270..5a6b2501 100644 --- a/canyonos_core/Dockerfile +++ b/canyonos_core/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.11-slim ENV PYTHONUNBUFFERED=1 -RUN apt-get update && apt-get install -y docker.io && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y docker.io zstd && rm -rf /var/lib/apt/lists/* COPY . /src RUN pip install /src diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/README.md b/canyonos_core/controller/cloud_provider_logic/EC2/README.md index 500e1755..a2999b37 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/README.md +++ b/canyonos_core/controller/cloud_provider_logic/EC2/README.md @@ -88,7 +88,7 @@ ec2: subnet_id: subnet-0123456789abcdef0 security_group_ids: - sg-0123456789abcdef0 - + # Optional Commands instance_profile_name: ec2launch # For the launched instances, if you want a specific IAM role attached to each instance, pass this variable in. ami_id: ami-0123456789abcdef0 # Look at `Creating your own AMI` below @@ -114,8 +114,17 @@ For the instance_profile_name, you will also need to add a new policy to your IA ## [OPTIONAL] Private Key -By default, canyonos creates a new private key for every project you deploy (The key pair is named `canyonos-ec2--`), but you can set your own key pair in the global_controller.yaml file in the ec2 block. -In addition to the security group and subnet_id, if you place `ssh_private_key_path: path/to/private/key` there, the key used will instead be that. +`ec2.ssh_private_key_path` is optional, defaulting to `~/.ssh/canyonos_ec2`. +If nothing exists at that default path, canyonos generates a fresh ed25519 +keypair there for you on first use (readable only by its owner: `chmod 600`). +If you set `ec2.ssh_private_key_path` to your own path instead, that file +must already exist — canyonos only auto-generates the default, never a +path you explicitly configured. + +The AWS-side key pair is always automatic either way: canyonos imports +whichever key you end up with (generated or your own) into AWS on first use, +named `canyonos-ec2--`, so the AMI never needs the +key pre-authorized. ## [OPTIONAL] Creating your own AMI diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py b/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py index 5f98b68b..8a867fa9 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py +++ b/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py @@ -19,6 +19,7 @@ import socket import stat import subprocess +import threading import time from typing import Any @@ -36,22 +37,44 @@ PROVIDER = "ec2" PUBLIC_IP_TIMEOUT = 120 CONTROLLER_HEALTH_TIMEOUT = 180 -DEFAULT_SSH_KEY_PATH = os.path.expanduser("~/.ssh/ventis_ec2") DEFAULT_SSH_USER = "ubuntu" +DEFAULT_SSH_KEY_PATH = os.path.expanduser("~/.ssh/canyonos_ec2") # This default AMI is a public AMI created by Canyon Code, containing base Ubuntu + Docker + zstd. Can be overriden manually with your own ami_id DEFAULT_AMI_ID = "ami-0101d5f2a2a9cd55c" _controller: Any = None +_default_key_lock = threading.Lock() + + +def _generate_default_key(key_path): + """Create a fresh ed25519 keypair at the default SSH key path. + + Guarded by a lock since replicas provision concurrently in a thread pool + -- without it, two threads could both see the file missing and race to + generate it at the same path. + """ + with _default_key_lock: + if os.path.isfile(key_path): + return + os.makedirs(os.path.dirname(key_path), exist_ok=True, mode=0o700) + subprocess.run( + ["ssh-keygen", "-t", "ed25519", "-N", "", "-f", key_path, "-q"], + check=True, + ) + os.chmod(key_path, 0o600) def _ssh_key_path(cfg): """Return the configured EC2 SSH identity after validating it locally.""" - key_path = os.path.expanduser(cfg.get("ssh_private_key_path", DEFAULT_SSH_KEY_PATH)) + configured_path = cfg.get("ssh_private_key_path") + key_path = os.path.expanduser(configured_path or DEFAULT_SSH_KEY_PATH) if not os.path.isfile(key_path): - raise ValueError( - f"EC2 SSH private key does not exist: {key_path}. " - "Set ec2.ssh_private_key_path to the controller key." - ) + if configured_path: + raise ValueError( + f"EC2 SSH private key does not exist: {key_path}. " + "Set ec2.ssh_private_key_path to the controller key." + ) + _generate_default_key(key_path) if not os.access(key_path, os.R_OK): raise ValueError(f"EC2 SSH private key is not readable: {key_path}") diff --git a/canyonos_core/controller/global_controller.py b/canyonos_core/controller/global_controller.py index 72bdc4e5..50bacfb3 100644 --- a/canyonos_core/controller/global_controller.py +++ b/canyonos_core/controller/global_controller.py @@ -810,7 +810,7 @@ def _send(instance): def _ssh_args(self, host, user=None): """Return the `ssh ... target` prefix used to reach a remote host.""" ssh_key_path = os.path.expanduser( - self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/ventis_ec2") + self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/canyonos_ec2") ) return [ "ssh", diff --git a/examples/helloworld/config/global_controller.yaml b/examples/helloworld/config/global_controller.yaml index 8b24a496..a3643ca9 100644 --- a/examples/helloworld/config/global_controller.yaml +++ b/examples/helloworld/config/global_controller.yaml @@ -55,4 +55,4 @@ ec2: - sg-0123456789abcdef0 ssh_user: ubuntu # Private key on the global controller used to SSH to EC2 workers. - ssh_private_key_path: ~/.ssh/ventis_ec2 + ssh_private_key_path: ~/.ssh/canyonos_ec2 diff --git a/tests/test_runtime_ec2.py b/tests/test_runtime_ec2.py index 633246c6..c23d0b7b 100644 --- a/tests/test_runtime_ec2.py +++ b/tests/test_runtime_ec2.py @@ -1,7 +1,9 @@ import os +import stat import subprocess import sys import tempfile +import threading import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -136,6 +138,52 @@ def test_aws_clients_defaults_ssh_user_when_missing(self): self.assertEqual(cfg["ssh_user"], ec2_runtime.DEFAULT_SSH_USER) + def test_aws_clients_generates_default_key_when_unset(self): + self.controller.config["ec2"].pop("ssh_private_key_path") + default_dir = tempfile.mkdtemp() + default_path = os.path.join(default_dir, "canyonos_ec2") + + try: + with patch.object(ec2_runtime, "DEFAULT_SSH_KEY_PATH", default_path): + cfg, _ = ec2_runtime._aws_clients() + self.assertEqual( + ec2_runtime._ssh_key_path(self.controller.config["ec2"]), + default_path, + ) + + self.assertTrue(os.path.isfile(default_path)) + mode = stat.S_IMODE(os.stat(default_path).st_mode) + self.assertEqual(mode, 0o600) + finally: + os.unlink(default_path) + if os.path.exists(f"{default_path}.pub"): + os.unlink(f"{default_path}.pub") + os.rmdir(default_dir) + + def test_default_key_generation_is_race_safe(self): + default_dir = tempfile.mkdtemp() + default_path = os.path.join(default_dir, "canyonos_ec2") + + try: + with patch.object(ec2_runtime, "DEFAULT_SSH_KEY_PATH", default_path): + threads = [ + threading.Thread( + target=ec2_runtime._generate_default_key, args=(default_path,) + ) + for _ in range(5) + ] + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertTrue(os.path.isfile(default_path)) + finally: + os.unlink(default_path) + if os.path.exists(f"{default_path}.pub"): + os.unlink(f"{default_path}.pub") + os.rmdir(default_dir) + def test_aws_clients_rejects_missing_ssh_private_key(self): self.controller.config["ec2"]["ssh_private_key_path"] = ( "/tmp/missing-canyonos-key" From acf0b6b235ded9f7a4f94ac71a5792e047789d63 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 18 Sep 2026 13:17:48 -0700 Subject: [PATCH 7/7] EC2 hardened --- .../cloud_provider_logic/EC2/README.md | 4 +--- .../cloud_provider_logic/EC2/_runtime.py | 10 ++++++---- .../controller/utils/redis_client.py | 20 ++++++++++++++++++- cli/canyonos/deploy.py | 2 +- examples/portfolio/.env.example | 5 +++++ .../portfolio/config/global_controller.yaml | 19 +++++++++--------- tests/test_runtime_ec2.py | 17 ++++++++++++---- 7 files changed, 55 insertions(+), 22 deletions(-) diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/README.md b/canyonos_core/controller/cloud_provider_logic/EC2/README.md index a2999b37..2b3afcda 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/README.md +++ b/canyonos_core/controller/cloud_provider_logic/EC2/README.md @@ -16,7 +16,7 @@ container there. - **Security group** (everything is inbound, same permissions needed for outbound unless you allow all outbound traffic) - The workflow and dashboard port need to be opened up, they are by default 8080 and 8081, and are referred as such below - - Port 50051, 6379, 22 needs to be accessible by the security group + - Port 50051, 6379, and 22 needs to be accessible by the security group, as well as wherever the host runs. - Port 8080 needs be accessible by whoever queries the workflow (0.0.0.0 for example) ## Config (`ec2:` block in `global_controller.yaml`) @@ -110,8 +110,6 @@ For the instance_profile_name, you will also need to add a new policy to your IA } ``` - - ## [OPTIONAL] Private Key `ec2.ssh_private_key_path` is optional, defaulting to `~/.ssh/canyonos_ec2`. diff --git a/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py b/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py index 8a867fa9..8ea3e9f3 100644 --- a/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py +++ b/canyonos_core/controller/cloud_provider_logic/EC2/_runtime.py @@ -182,13 +182,13 @@ def provision_instance(spec, replica_index, next_host_port=None): if instance: break if instance and ( - instance.get("PrivateIpAddress") or instance.get("PublicIpAddress") + instance.get("PublicIpAddress") or instance.get("PrivateIpAddress") ): break time.sleep(2) host = ( - instance.get("PrivateIpAddress") or instance.get("PublicIpAddress") + instance.get("PublicIpAddress") or instance.get("PrivateIpAddress") if instance else None ) @@ -196,8 +196,10 @@ def provision_instance(spec, replica_index, next_host_port=None): raise RuntimeError( f"EC2 instance {instance_id} does not have a reachable IP address." ) - # Kept alongside `host` (a private IP inside the VPC): callers outside the - # VPC, such as the CLI printing where to send requests, need this one. + # Public preferred: the deploying host (the machine running `canyonos + # deploy`) is outside the VPC in the common case, so SSH/build traffic + # needs the public IP. Kept alongside `host` for callers that want it + # explicitly regardless of which one `host` ended up being. public_host = instance.get("PublicIpAddress") if instance else None redis_port = spec.get( diff --git a/canyonos_core/controller/utils/redis_client.py b/canyonos_core/controller/utils/redis_client.py index 9b52d8ee..e5eca893 100644 --- a/canyonos_core/controller/utils/redis_client.py +++ b/canyonos_core/controller/utils/redis_client.py @@ -1,11 +1,29 @@ +import socket + import redis +def _prefer_ipv4(host): + """Resolve `host` to an IPv4 address when one exists. + + Some hosts (e.g. Docker Desktop's host.docker.internal) advertise both an + IPv4 and an unreachable IPv6 address for the same name; redis-py has no + address-family fallback, so it fails outright if it happens to try the + broken one first. Falls back to the original host if resolution fails + (already an IP, unresolvable, IPv4-only environments, etc). + """ + try: + infos = socket.getaddrinfo(host, None, socket.AF_INET) + except socket.gaierror: + return host + return infos[0][4][0] if infos else host + + class RedisClient(object): """Redis utility for connecting to localhost with support for strings, hashes, and sets.""" def __init__(self, host="localhost", port=6379, db=0): - self.client = redis.Redis(host=host, port=port, db=db) + self.client = redis.Redis(host=_prefer_ipv4(host), port=port, db=db) # --- String operations --- diff --git a/cli/canyonos/deploy.py b/cli/canyonos/deploy.py index 8f834393..116bbf58 100644 --- a/cli/canyonos/deploy.py +++ b/cli/canyonos/deploy.py @@ -280,7 +280,7 @@ def _summary_body(dashboard_url, targets, config_path): body.append("\n") body.append(_curl_example(f"{base}/{route}", example), WHITE) body.append("\npoll ", "dim") - body.append(f"{base}/status/", WHITE) + body.append(f"curl {base}/status/", WHITE) if host not in ("127.0.0.1", "localhost"): body.append(f"\n needs inbound TCP {port} open on {host}", "dim") return body diff --git a/examples/portfolio/.env.example b/examples/portfolio/.env.example index b8e6c8fe..8d3a531a 100644 --- a/examples/portfolio/.env.example +++ b/examples/portfolio/.env.example @@ -12,6 +12,11 @@ AWS_BEARER_TOKEN_BEDROCK= # .claude/skills/porting-to-canyonos/references/llm-proxy.md. AWS_ENDPOINT_URL_BEDROCK_RUNTIME=http://host.docker.internal:8081/bedrock +# --- AWS EC2 (provider: EC2 agents, config/global_controller.yaml's ec2: block) --- +EC2_REGION= +EC2_SUBNET_ID= +EC2_SECURITY_GROUP_ID= + # --- CanyonOS controller / web --- CANYONOS_API_IMAGE= CANYONOS_WEB_IMAGE= diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index 8e6ae1aa..e2530497 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -16,7 +16,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/intent_agent.py - provider: local + provider: Local instance_type: t3.micro # Stage 0b: price history fetch. Network/IO-bound, cheap CPU. Called by @@ -28,7 +28,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/price_agent.py - provider: local + provider: Local instance_type: t3.micro requirements: [yfinance] @@ -41,7 +41,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/metrics_agent.py - provider: local + provider: Local instance_type: t3.micro # Stage 2: portfolio-level risk aggregation. Single call per request; needs @@ -53,7 +53,7 @@ agents: cpu: 1 memory: 256 entrypoint: agents/risk_agent.py - provider: local + provider: Local instance_type: t3.micro # Stage 3: LLM briefing via Bedrock. On the critical path, one call per @@ -65,7 +65,7 @@ agents: cpu: 1 memory: 512 entrypoint: agents/advisor_agent.py - provider: local + provider: Local instance_type: t3.micro # The workflow, exposed as a REST API. @@ -76,7 +76,7 @@ agents: redis_port: 6379 replicas: 1 workflow_file: workflow/portfolio_workflow.py - provider: local + provider: Local instance_type: t3.micro @@ -93,6 +93,10 @@ otel: # Polling interval in seconds poll_interval: 4 +# Hands each agent container the project's .env (AWS credentials, Bedrock +# bearer token, etc.) via `docker run --env-file`. +env_file: .env + redis: host: localhost port: 6379 @@ -101,10 +105,7 @@ redis: # EC2 defaults for `provider: EC2` replicas. ec2: region: ${EC2_REGION} - ami_id: ${EC2_AMI_ID} subnet_id: ${EC2_SUBNET_ID} security_group_ids: - ${EC2_SECURITY_GROUP_ID} - ssh_user: ${EC2_SSH_USER} - ssh_private_key_path: ${EC2_SSH_PRIVATE_KEY_PATH} diff --git a/tests/test_runtime_ec2.py b/tests/test_runtime_ec2.py index c23d0b7b..2fb4b445 100644 --- a/tests/test_runtime_ec2.py +++ b/tests/test_runtime_ec2.py @@ -256,12 +256,21 @@ def test_provision_uses_ec2_client(self): {"Key": "Name", "Value": "canyonos-Tagged-2"}, ) self.assertEqual(self.fake_client.waiter.calls, [["i-test1"]]) - self.assertEqual(provisioned["host"], "10.0.0.30") + self.assertEqual(provisioned["host"], "54.10.20.30") self.assertEqual( self.client_calls, [{"service_name": "ec2", "region_name": "us-east-1"}], ) + def test_provision_falls_back_to_private_ip_when_no_public_ip(self): + self.fake_client.public_ip = None + spec = {"name": "Tagged", "provider": "EC2", "instance_type": "t3.small"} + + provisioned = ec2_runtime.provision_instance(spec, 0) + + self.assertEqual(provisioned["host"], "10.0.0.30") + self.assertIsNone(provisioned["public_host"]) + def test_provision_attaches_instance_profile_when_configured(self): self.controller.config["ec2"]["instance_profile_name"] = "my-custom-profile" spec = {"name": "Tagged", "provider": "EC2", "instance_type": "t3.small"} @@ -288,9 +297,9 @@ def test_provision_and_bootstrap_instance_return_runtime_record(self): provisioned = ec2_runtime.provision_instance(spec, 2) instance = ec2_runtime.bootstrap_instance(provisioned, spec, 2, "agent-id-2") - self.assertEqual(instance["host"], "10.0.0.30") - self.assertEqual(instance["endpoint"], "10.0.0.30:50051") - self.assertEqual(instance["redis_host"], "10.0.0.30") + self.assertEqual(instance["host"], "54.10.20.30") + self.assertEqual(instance["endpoint"], "54.10.20.30:50051") + self.assertEqual(instance["redis_host"], "54.10.20.30") self.assertEqual(instance["redis_port"], "6390") self.assertIn("--i-test1", instance["runtime_id"])