diff --git a/Makefile b/Makefile index 26f593412..173f6e06a 100644 --- a/Makefile +++ b/Makefile @@ -161,7 +161,15 @@ export MORPH_RETH_RUSTFLAGS export MORPH_RETH_DOCKER_TARGET export MORPH_RETH_ENTRYPOINT DEVNET_COMPOSE_FILES := -f docker-compose-devnet.yml -DEVNET_CLEAN_COMPOSE_FILES := -f docker-compose-devnet.yml -f docker-compose-reth.yml -f docker-compose-cluster.yml +DEVNET_CLEAN_COMPOSE_FILES := -f docker-compose-devnet.yml -f docker-compose-cluster.yml -f docker-compose-reth.yml + +# The cluster topology is layered before the execution-client override, so that +# docker-compose-reth.yml has the last word on the ha-el-* image, entrypoint and +# command. Swapping these two leaves the cluster nodes running geth even when +# reth was asked for, because later -f files win. +ifneq ($(DEVNET_CLUSTER_ENABLED),) +DEVNET_COMPOSE_FILES += -f docker-compose-cluster.yml +endif ifeq ($(EXECUTION_CLIENT),geth) DEVNET_EXECUTION_DEPS := submodules @@ -175,9 +183,6 @@ endif else $(error unsupported EXECUTION_CLIENT "$(EXECUTION_CLIENT)", expected "geth" or "reth") endif -ifneq ($(DEVNET_CLUSTER_ENABLED),) -DEVNET_COMPOSE_FILES += -f docker-compose-cluster.yml -endif devnet-up: $(DEVNET_EXECUTION_DEPS) go-ubuntu-builder python3 ops/devnet-morph/main.py --polyrepo-dir=. --execution-client=$(EXECUTION_CLIENT) \ diff --git a/ops/README.md b/ops/README.md index b286c19eb..183209a39 100644 --- a/ops/README.md +++ b/ops/README.md @@ -84,10 +84,57 @@ looking anywhere else. | L1 | `9545` | `9546` | beacon `4000` | | `morph-el-0` | `8545` | `8546` | `node-0` → `26657` | | `morph-el-1` | `8645` | `8646` | — | -| `ha-geth-0/1/2` | `9145` / `9245` / `9345` | `9146` / `9246` / `9346` | `27657` / `27757` / `27857` | +| `ha-el-0/1/2` | `9145` / `9245` / `9345` | `9146` / `9246` / `9346` | `27657` / `27757` / `27857` | `ha-node` admin API: `9501` / `9601` / `9701`. +The execution-layer services are named `el` rather than `geth` because either +client can back them: `docker-compose-cluster.yml` defines them as geth, and +`docker-compose-reth.yml` overrides them to reth. That override only works +because the cluster file is layered *before* the reth file — later `-f` files +win, so the reverse order silently leaves the cluster on geth. + +## Execution-layer peering + +Discovery is off everywhere (`--nodiscover` / `--disable-discovery`), so peers +are configured explicitly and the topology is fixed: + +- geth reads `static-nodes.json` (mounted into `morph-el-1`) and, for the + cluster, `static-nodes-cluster.json` (mounted into all three `ha-el-*`). +- reth ignores those files and takes `--trusted-peers` on the command line. + +Both clients derive their identity from the same `nodekey*` / `ha-nodekey*` +files, so a node's enode is the same whichever client is running. reth needs +`--p2p-secret-key` for this; without it, it invents a random identity per +datadir and no peer list can be written in advance. The key files must not have +a trailing newline — reth rejects those with `malformed or out-of-range secret +key`, while geth tolerates them either way. + +`morph-el-0` and `morph-el-1` only know each other. The `ha-el-*` nodes dial +both of those plus each other, which keeps `ha-el-*` names out of the +non-cluster setup, where they would not resolve. + +## Consensus-layer peering + +`setup_nodes.py` writes `persistent_peers` for every tendermint home, deriving +each node ID from the `node_key.json` that ends up installed — which is why the +key files are copied before the peer list is built. Overwriting a +`node_key.json` changes the node's identity, so a hardcoded ID silently goes +stale. + +The list contains only the nodes that actually run tendermint: `node-0` and the +three `ha-node-*`. `node-1` runs with +`MORPH_NODE_DERIVATION_VERIFY_MODE=layer1` and never starts tendermint, and +`node-2` has no compose service at all; listing either just produces endless +reconnect and DNS failures. + +The `ha-node-*` reaching each other matters: the sequencer hand-over waits for +the block pool to report caught up, and a pool whose only peers are unreachable +never does. That is the silent-stall-at-height-0 failure described above. + +RPC is served on `0.0.0.0:26657` inside each container so the published ports in +the table above are actually reachable from the host. + Each geth serves metrics on `6060` inside its container, with `--metrics.expensive` on. Without that flag every counter behind `metrics.EnabledExpensive` stays zero, which blanks `chain/account/*` and diff --git a/ops/devnet-morph/devnet/__init__.py b/ops/devnet-morph/devnet/__init__.py index 530e52c59..5d97b2e5d 100644 --- a/ops/devnet-morph/devnet/__init__.py +++ b/ops/devnet-morph/devnet/__init__.py @@ -66,10 +66,14 @@ def compose_file_args(execution_client, cluster=False): """Return docker-compose -f flags for the chosen L2 execution client.""" args = ['-f', 'docker-compose-devnet.yml'] - if execution_client == 'reth': - args.extend(['-f', 'docker-compose-reth.yml']) + # The cluster topology comes before the execution-client override so that + # docker-compose-reth.yml gets the last word on the ha-el-* image, + # entrypoint and command. Later -f files win, so reversing these two leaves + # the cluster nodes on geth even when reth was requested. if cluster: args.extend(['-f', 'docker-compose-cluster.yml']) + if execution_client == 'reth': + args.extend(['-f', 'docker-compose-reth.yml']) return args diff --git a/ops/devnet-morph/devnet/setup_nodes.py b/ops/devnet-morph/devnet/setup_nodes.py index d2f2f4a54..827394f48 100644 --- a/ops/devnet-morph/devnet/setup_nodes.py +++ b/ops/devnet-morph/devnet/setup_nodes.py @@ -1,9 +1,109 @@ +import base64 +import hashlib +import json import os import shutil import subprocess import sys import re +# Directories that hold a node's tendermint home, in the order their config +# files are processed. The first entry is the genesis validator. +NODE_DIRS = ("node0", "node1", "node2", "ha-node0", "ha-node1", "ha-node2") + +# Directory name -> compose service hostname, used to build peer addresses. +SERVICE_HOSTNAMES = { + "node0": "node-0", + "node1": "node-1", + "node2": "node-2", + "ha-node0": "ha-node-0", + "ha-node1": "ha-node-1", + "ha-node2": "ha-node-2", +} + +# Nodes that actually run tendermint, and so can be dialed as peers. node-1 +# runs with MORPH_NODE_DERIVATION_VERIFY_MODE=layer1 and never starts +# tendermint; node-2 has no compose service at all. Listing either as a peer +# only produces endless reconnect and DNS lookup failures. +TENDERMINT_PEERS = ("node0", "ha-node0", "ha-node1", "ha-node2") + + +def tendermint_node_id(node_key_path): + """Derive a tendermint node ID from a node_key.json file. + + The ID is the hex encoding of the first 20 bytes of sha256(pubkey). An + ed25519 private key is stored as seed(32) || pubkey(32), so the public half + is the tail of the decoded value. + + IDs must be derived from the key files that are actually in place, which is + why this runs after the key files have been copied: overwriting a + node_key.json changes the node's identity. + """ + with open(node_key_path) as f: + priv_key = json.load(f)["priv_key"]["value"] + pubkey = base64.b64decode(priv_key)[32:] + return hashlib.sha256(pubkey).hexdigest()[:40] + + +def copy_key_files(docker_dir, devnet_dir): + """Install the fixed node keys and the shared genesis into each node home. + + Only node0 gets a genesis validator key. The others must not have one: a + node holding the sole genesis validator key gets block sync disabled and + never hands over to the sequencer routines. + """ + print("Copying key files...") + + for node in NODE_DIRS: + source_dir = os.path.join(docker_dir, node) + dest_dir = os.path.join(devnet_dir, node, "config") + + if not os.path.isdir(dest_dir): + print(f"Error: Missing destination directory for {node}. Exiting.") + sys.exit(1) + + if os.path.isdir(source_dir): + shutil.copyfile(os.path.join(source_dir, "node_key.json"), os.path.join(dest_dir, "node_key.json")) + + if node == "node0" and os.path.isdir(source_dir): + shutil.copyfile(os.path.join(source_dir, "priv_validator_key.json"), os.path.join(dest_dir, "priv_validator_key.json")) + else: + priv_validator_key = os.path.join(dest_dir, "priv_validator_key.json") + priv_validator_state = os.path.join(devnet_dir, node, "data", "priv_validator_state.json") + for validator_file in (priv_validator_key, priv_validator_state): + if os.path.exists(validator_file): + os.remove(validator_file) + + # Copy and rename genesis file + shutil.copyfile(os.path.join(docker_dir, "tendermint-devnet-genesis.json"), os.path.join(dest_dir, "genesis.json")) + + print(f"Files copied successfully for {node}.") + + print("All key files have been copied successfully.") + + +def build_persistent_peers(devnet_dir): + """Map each node directory to the peer list it should dial. + + Every tendermint-running node is given all the others, so the HA nodes + reach each other rather than only node-0. Without that the HA block pool + never reports caught up, the sequencer hand-over never runs, and the + cluster silently stalls at height 0. + """ + addresses = {} + for node in TENDERMINT_PEERS: + node_key = os.path.join(devnet_dir, node, "config", "node_key.json") + node_id = tendermint_node_id(node_key) + addresses[node] = f"{node_id}@{SERVICE_HOSTNAMES[node]}:26656" + + peers = {} + for node in NODE_DIRS: + peers[node] = ",".join( + address for peer, address in addresses.items() if peer != node + ) + return peers + + def setup_devnet_nodes(): """ Set up the devnet nodes, modify configuration files using toml library, and copy key files. @@ -33,14 +133,7 @@ def setup_devnet_nodes(): devnet_dir = os.path.join(docker_dir, ".devnet") if os.path.exists(devnet_dir): old_topology_paths = [os.path.join(devnet_dir, f"node{i}") for i in range(3, 6)] - expected_paths = [ - os.path.join(devnet_dir, "node0"), - os.path.join(devnet_dir, "node1"), - os.path.join(devnet_dir, "node2"), - os.path.join(devnet_dir, "ha-node0"), - os.path.join(devnet_dir, "ha-node1"), - os.path.join(devnet_dir, "ha-node2"), - ] + expected_paths = [os.path.join(devnet_dir, node) for node in NODE_DIRS] if any(os.path.exists(path) for path in old_topology_paths) or any( not os.path.exists(path) for path in expected_paths): print("Existing stale devnet detected. Regenerating single-sequencer config.") @@ -73,20 +166,17 @@ def setup_devnet_nodes(): if os.path.exists(generated_path): os.rename(generated_path, desired_path) + # Install the key files first: node IDs are derived from node_key.json, so + # the peer addresses below must be computed from the final keys. + copy_key_files(docker_dir, devnet_dir) + + persistent_peers = build_persistent_peers(devnet_dir) + # Modify config.toml files. print("Modifying config.toml files...") - config_files = [ - os.path.join(devnet_dir, node, "config", "config.toml") - for node in ("node0", "node1", "node2", "ha-node0", "ha-node1", "ha-node2") - ] - - persistent_peers_value = ( - "93e27ea2306e158a8146d5f44caaab97496797d2@node-0:26656," - "7f78b7d7a7e6bad4faf68d5731d437f4288d96d0@node-1:26656," - "06c699be2f9aeb9f7ec79f508a95ff80576deb12@node-2:26656" - ) - for i, config_file in enumerate(config_files): + for i, node in enumerate(NODE_DIRS): + config_file = os.path.join(devnet_dir, node, "config", "config.toml") if not os.path.isfile(config_file): print(f"Error: {config_file} not found. Exiting.") sys.exit(1) @@ -102,7 +192,11 @@ def setup_devnet_nodes(): content = content.replace('send_rate = 5120000', 'send_rate = 52428800') content = content.replace('recv_rate = 5120000', 'recv_rate = 102428800') content = content.replace('block_sync = false', 'block_sync = true') - content = re.sub(r'persistent_peers\s*=\s*".*?"', f'persistent_peers = "{persistent_peers_value}"', content) + content = re.sub(r'persistent_peers\s*=\s*".*?"', f'persistent_peers = "{persistent_peers[node]}"', content) + + # Serve the RPC on all interfaces so the published container ports + # (26657, 27657, 27757, 27857) are reachable from the host. + content = content.replace('laddr = "tcp://127.0.0.1:26657"', 'laddr = "tcp://0.0.0.0:26657"') # Modify pex for the sequencer validator node. if i == 0: @@ -115,35 +209,4 @@ def setup_devnet_nodes(): f.write(content) print("All config.toml files have been updated successfully.") - - # Copy key files to devnet node directories - print("Copying key files...") - node_dirs = ["node0", "node1", "node2", "ha-node0", "ha-node1", "ha-node2"] - - for node in node_dirs: - source_dir = os.path.join(docker_dir, node) - dest_dir = os.path.join(devnet_dir, node, "config") - - if not os.path.isdir(dest_dir): - print(f"Error: Missing destination directory for {node}. Exiting.") - sys.exit(1) - - if os.path.isdir(source_dir): - shutil.copyfile(os.path.join(source_dir, "node_key.json"), os.path.join(dest_dir, "node_key.json")) - - if node == "node0" and os.path.isdir(source_dir): - shutil.copyfile(os.path.join(source_dir, "priv_validator_key.json"), os.path.join(dest_dir, "priv_validator_key.json")) - else: - priv_validator_key = os.path.join(dest_dir, "priv_validator_key.json") - priv_validator_state = os.path.join(devnet_dir, node, "data", "priv_validator_state.json") - for validator_file in (priv_validator_key, priv_validator_state): - if os.path.exists(validator_file): - os.remove(validator_file) - - # Copy and rename genesis file - shutil.copyfile(os.path.join(docker_dir, "tendermint-devnet-genesis.json"), os.path.join(dest_dir, "genesis.json")) - - print(f"Files copied successfully for {node}.") - - print("All key files have been copied successfully.") print("Devnet nodes setup completed successfully.") diff --git a/ops/devnet-morph/tests/test_devnet_config.py b/ops/devnet-morph/tests/test_devnet_config.py index 7881fe7b9..1b88cc38c 100644 --- a/ops/devnet-morph/tests/test_devnet_config.py +++ b/ops/devnet-morph/tests/test_devnet_config.py @@ -43,7 +43,7 @@ def test_devnet_clean_removes_compose_project_volumes(self): makefile = (REPO_ROOT / "Makefile").read_text() self.assertIn( - "DEVNET_CLEAN_COMPOSE_FILES := -f docker-compose-devnet.yml -f docker-compose-reth.yml -f docker-compose-cluster.yml", + "DEVNET_CLEAN_COMPOSE_FILES := -f docker-compose-devnet.yml -f docker-compose-cluster.yml -f docker-compose-reth.yml", makefile, ) self.assertIn("docker compose $(DEVNET_CLEAN_COMPOSE_FILES) down --volumes --remove-orphans", makefile) @@ -118,11 +118,87 @@ def test_cluster_compose_defines_ha_services(self): self.assertTrue(cluster_compose.exists()) compose = cluster_compose.read_text() - for service in ("ha-geth-0:", "ha-geth-1:", "ha-geth-2:", "ha-node-0:", "ha-node-1:", "ha-node-2:"): + for service in ("ha-el-0:", "ha-el-1:", "ha-el-2:", "ha-node-0:", "ha-node-1:", "ha-node-2:"): self.assertIn(service, compose) + self.assertNotIn("ha-geth", compose) self.assertIn("MORPH_NODE_HA_ENABLED=true", compose) self.assertIn("MORPH_NODE_HA_BOOTSTRAP=true", compose) self.assertIn("MORPH_NODE_HA_JOIN=ha-node-0:9401", compose) + for index in (0, 1, 2): + self.assertIn(f"MORPH_NODE_L2_ETH_RPC=http://ha-el-{index}:8545", compose) + self.assertIn(f"MORPH_NODE_L2_ENGINE_RPC=http://ha-el-{index}:8551", compose) + + def test_cluster_geth_nodes_peer_with_each_other(self): + """The shared static-nodes.json lists only morph-el-*, so the cluster + needs its own file or the ha-el nodes never dial each other.""" + cluster_static_nodes = (DOCKER_DIR / "static-nodes-cluster.json").read_text() + + for host in ("ha-el-0", "ha-el-1", "ha-el-2", "morph-el-0", "morph-el-1"): + self.assertIn(f"@{host}:30303", cluster_static_nodes) + + compose = (DOCKER_DIR / "docker-compose-cluster.yml").read_text() + # geth only reads the fixed filename, so the mount has to be renamed on + # the way in. + self.assertIn( + '"${PWD}/static-nodes-cluster.json:/db/geth/static-nodes.json"', + compose, + ) + + def test_reth_compose_configures_deterministic_peering(self): + compose = (DOCKER_DIR / "docker-compose-reth.yml").read_text() + + # reth cannot read geth's static-nodes.json, so peers are passed as + # flags, and a fixed key file is what makes the enodes predictable. + for service, key in ( + ("morph-el-0", "nodekey0"), + ("morph-el-1", "nodekey1"), + ("ha-el-0", "ha-nodekey0"), + ("ha-el-1", "ha-nodekey1"), + ("ha-el-2", "ha-nodekey2"), + ): + self.assertIn(f"{service}:", compose) + self.assertIn(f'"${{PWD}}/{key}:/p2p-secret.key"', compose) + self.assertEqual(compose.count("--p2p-secret-key=/p2p-secret.key"), 5) + self.assertEqual(compose.count("--trusted-peers="), 5) + + def test_execution_client_keys_have_no_trailing_newline(self): + """reth rejects a key file with a trailing newline ("malformed or + out-of-range secret key"); geth accepts it either way.""" + for key in ("nodekey0", "nodekey1", "nodekey2", + "ha-nodekey0", "ha-nodekey1", "ha-nodekey2"): + contents = (DOCKER_DIR / key).read_bytes() + self.assertEqual(len(contents), 64, f"{key} should be 64 hex characters") + self.assertFalse(contents.endswith(b"\n"), f"{key} must not end with a newline") + + def test_tendermint_peers_are_derived_from_installed_node_keys(self): + sys.path.insert(0, str(DEVNET_PACKAGE)) + try: + setup_nodes = importlib.import_module("devnet.setup_nodes") + importlib.reload(setup_nodes) + finally: + sys.path.remove(str(DEVNET_PACKAGE)) + + # Anchored on the ID that used to be hardcoded in persistent_peers. + self.assertEqual( + setup_nodes.tendermint_node_id(DOCKER_DIR / "node0" / "node_key.json"), + "93e27ea2306e158a8146d5f44caaab97496797d2", + ) + + # node-1 never starts tendermint and node-2 has no compose service, so + # neither may appear as a peer. + self.assertEqual( + setup_nodes.TENDERMINT_PEERS, + ("node0", "ha-node0", "ha-node1", "ha-node2"), + ) + + source = (DEVNET_PACKAGE / "devnet" / "setup_nodes.py").read_text() + # Node IDs come from the keys that end up installed, so the copy step + # must run before the peer list is built. + self.assertLess( + source.index("copy_key_files(docker_dir, devnet_dir)"), + source.index("persistent_peers = build_persistent_peers(devnet_dir)"), + ) + self.assertIn('laddr = "tcp://0.0.0.0:26657"', source) def test_compose_file_args_can_enable_cluster_mode(self): sys.path.insert(0, str(DEVNET_PACKAGE)) @@ -141,15 +217,18 @@ def test_compose_file_args_can_enable_cluster_mode(self): devnet.compose_file_args("geth", cluster=True), ["-f", "docker-compose-devnet.yml", "-f", "docker-compose-cluster.yml"], ) + # The cluster file must come first so the reth override wins on the + # ha-el-* image, entrypoint and command; reversed, the cluster silently + # stays on geth. self.assertEqual( devnet.compose_file_args("reth", cluster=True), [ "-f", "docker-compose-devnet.yml", "-f", - "docker-compose-reth.yml", - "-f", "docker-compose-cluster.yml", + "-f", + "docker-compose-reth.yml", ], ) diff --git a/ops/docker/docker-compose-cluster.yml b/ops/docker/docker-compose-cluster.yml index f5c75e6fb..c4fd14fe4 100644 --- a/ops/docker/docker-compose-cluster.yml +++ b/ops/docker/docker-compose-cluster.yml @@ -1,6 +1,6 @@ services: - ha-geth-0: - container_name: ha-geth-0 + ha-el-0: + container_name: ha-el-0 depends_on: morph-el-0: condition: service_started @@ -17,15 +17,15 @@ services: - "${PWD}/jwt-secret.txt:/jwt-secret.txt" - "${PWD}/../l2-genesis/.devnet/genesis-l2.json:/genesis.json" - "${PWD}/ha-nodekey0:/db/geth/nodekey" - - "${PWD}/static-nodes.json:/db/geth/static-nodes.json" + - "${PWD}/static-nodes-cluster.json:/db/geth/static-nodes.json" environment: - RUST_LOG=${RUST_LOG} entrypoint: - "/bin/bash" - "/entrypoint.sh" - ha-geth-1: - container_name: ha-geth-1 + ha-el-1: + container_name: ha-el-1 depends_on: morph-el-0: condition: service_started @@ -42,15 +42,15 @@ services: - "${PWD}/jwt-secret.txt:/jwt-secret.txt" - "${PWD}/../l2-genesis/.devnet/genesis-l2.json:/genesis.json" - "${PWD}/ha-nodekey1:/db/geth/nodekey" - - "${PWD}/static-nodes.json:/db/geth/static-nodes.json" + - "${PWD}/static-nodes-cluster.json:/db/geth/static-nodes.json" environment: - RUST_LOG=${RUST_LOG} entrypoint: - "/bin/bash" - "/entrypoint.sh" - ha-geth-2: - container_name: ha-geth-2 + ha-el-2: + container_name: ha-el-2 depends_on: morph-el-0: condition: service_started @@ -67,7 +67,7 @@ services: - "${PWD}/jwt-secret.txt:/jwt-secret.txt" - "${PWD}/../l2-genesis/.devnet/genesis-l2.json:/genesis.json" - "${PWD}/ha-nodekey2:/db/geth/nodekey" - - "${PWD}/static-nodes.json:/db/geth/static-nodes.json" + - "${PWD}/static-nodes-cluster.json:/db/geth/static-nodes.json" environment: - RUST_LOG=${RUST_LOG} entrypoint: @@ -77,7 +77,7 @@ services: ha-node-0: container_name: ha-node-0 depends_on: - ha-geth-0: + ha-el-0: condition: service_started node-0: condition: service_started @@ -92,8 +92,8 @@ services: environment: - MORPH_NODE_SEQUENCER_PRIVATE_KEY=${SEQUENCER_PRIVATE_KEY} - MORPH_NODE_SEQUENCER_UPGRADE_TIME=${SEQUENCER_UPGRADE_TIME:-0} - - MORPH_NODE_L2_ETH_RPC=http://ha-geth-0:8545 - - MORPH_NODE_L2_ENGINE_RPC=http://ha-geth-0:8551 + - MORPH_NODE_L2_ETH_RPC=http://ha-el-0:8545 + - MORPH_NODE_L2_ENGINE_RPC=http://ha-el-0:8551 - MORPH_NODE_L2_ENGINE_AUTH=${JWT_SECRET_PATH} - MORPH_NODE_L1_ETH_RPC=${L1_ETH_RPC} - MORPH_NODE_L1_ETH_BEACON_RPC=${L1_BEACON_CHAIN_RPC} @@ -119,7 +119,7 @@ services: depends_on: ha-node-0: condition: service_started - ha-geth-1: + ha-el-1: condition: service_started image: morph-node:latest restart: unless-stopped @@ -132,8 +132,8 @@ services: environment: - MORPH_NODE_SEQUENCER_PRIVATE_KEY=${SEQUENCER_PRIVATE_KEY} - MORPH_NODE_SEQUENCER_UPGRADE_TIME=${SEQUENCER_UPGRADE_TIME:-0} - - MORPH_NODE_L2_ETH_RPC=http://ha-geth-1:8545 - - MORPH_NODE_L2_ENGINE_RPC=http://ha-geth-1:8551 + - MORPH_NODE_L2_ETH_RPC=http://ha-el-1:8545 + - MORPH_NODE_L2_ENGINE_RPC=http://ha-el-1:8551 - MORPH_NODE_L2_ENGINE_AUTH=${JWT_SECRET_PATH} - MORPH_NODE_L1_ETH_RPC=${L1_ETH_RPC} - MORPH_NODE_L1_ETH_BEACON_RPC=${L1_BEACON_CHAIN_RPC} @@ -159,7 +159,7 @@ services: depends_on: ha-node-0: condition: service_started - ha-geth-2: + ha-el-2: condition: service_started image: morph-node:latest restart: unless-stopped @@ -172,8 +172,8 @@ services: environment: - MORPH_NODE_SEQUENCER_PRIVATE_KEY=${SEQUENCER_PRIVATE_KEY} - MORPH_NODE_SEQUENCER_UPGRADE_TIME=${SEQUENCER_UPGRADE_TIME:-0} - - MORPH_NODE_L2_ETH_RPC=http://ha-geth-2:8545 - - MORPH_NODE_L2_ENGINE_RPC=http://ha-geth-2:8551 + - MORPH_NODE_L2_ETH_RPC=http://ha-el-2:8545 + - MORPH_NODE_L2_ENGINE_RPC=http://ha-el-2:8551 - MORPH_NODE_L2_ENGINE_AUTH=${JWT_SECRET_PATH} - MORPH_NODE_L1_ETH_RPC=${L1_ETH_RPC} - MORPH_NODE_L1_ETH_BEACON_RPC=${L1_BEACON_CHAIN_RPC} diff --git a/ops/docker/docker-compose-reth.yml b/ops/docker/docker-compose-reth.yml index 6d18823f4..749e961f1 100644 --- a/ops/docker/docker-compose-reth.yml +++ b/ops/docker/docker-compose-reth.yml @@ -1,32 +1,147 @@ -x-reth-command: &reth-command - - node - - --chain=/genesis.json - - --datadir=/db - - --http - - --http.addr=0.0.0.0 - - --http.port=8545 - - --http.api=web3,debug,eth,txpool,net,trace,admin,reth - - --ws - - --ws.addr=0.0.0.0 - - --ws.port=8546 - - --ws.api=web3,debug,eth,txpool,net,trace,admin,reth - - --authrpc.addr=0.0.0.0 - - --authrpc.port=8551 - - --authrpc.jwtsecret=/jwt-secret.txt - - --nat=none - - --disable-discovery +# reth does not read geth's static-nodes.json, so each execution client is +# handed its peers on the command line. Discovery is disabled, so --trusted-peers +# is the only thing wiring these nodes together. +# +# Every node is pointed at the same nodekey file geth mounts, via +# --p2p-secret-key, which makes its peer ID deterministic. Keep the two in sync: +# without the key file reth generates a fresh random identity for every new +# datadir, and no peer list could be written ahead of time. The enode public +# keys below are derived from those same files. +# +# Peering mirrors the geth static-nodes layout: morph-el-0 and morph-el-1 know +# only each other, while the HA nodes dial both of them and each other. That +# keeps ha-el-* out of the non-cluster setup, where those names do not resolve. +# +# The command list is spelled out per service because --trusted-peers takes one +# comma separated value and YAML cannot concatenate an anchor with extra items. x-reth-service: &reth-service image: ${MORPH_RETH_IMAGE:-ghcr.io/morph-l2/morph-reth:latest} user: "0:0" entrypoint: - ${MORPH_RETH_ENTRYPOINT:-/usr/local/bin/morph-reth} - command: *reth-command services: morph-el-0: <<: *reth-service build: !reset null + command: + - node + - --chain=/genesis.json + - --datadir=/db + - --http + - --http.addr=0.0.0.0 + - --http.port=8545 + - --http.api=web3,debug,eth,txpool,net,trace,admin,reth + - --ws + - --ws.addr=0.0.0.0 + - --ws.port=8546 + - --ws.api=web3,debug,eth,txpool,net,trace,admin,reth + - --authrpc.addr=0.0.0.0 + - --authrpc.port=8551 + - --authrpc.jwtsecret=/jwt-secret.txt + - --nat=none + - --disable-discovery + - --p2p-secret-key=/p2p-secret.key + - --trusted-peers=enode://bd755ce0bc8c06b4444b9013e8d1215a02e2b53f39f746f060c292ba2f6877d7b702374f006a49a7b1506bf1bc027b43824859d081283e6bac97c8600cdf3fee@morph-el-1:30303 + volumes: + - "${PWD}/nodekey0:/p2p-secret.key" morph-el-1: <<: *reth-service + command: + - node + - --chain=/genesis.json + - --datadir=/db + - --http + - --http.addr=0.0.0.0 + - --http.port=8545 + - --http.api=web3,debug,eth,txpool,net,trace,admin,reth + - --ws + - --ws.addr=0.0.0.0 + - --ws.port=8546 + - --ws.api=web3,debug,eth,txpool,net,trace,admin,reth + - --authrpc.addr=0.0.0.0 + - --authrpc.port=8551 + - --authrpc.jwtsecret=/jwt-secret.txt + - --nat=none + - --disable-discovery + - --p2p-secret-key=/p2p-secret.key + - --trusted-peers=enode://58e698ea2dd8a76e0cb185d13c1faabf223b60c89fef988c8b89496571056d6c2922109537bb291cd87f2ec09a23ac37d59bde2c7a4885d07b7b641cadff2921@morph-el-0:30303 + volumes: + - "${PWD}/nodekey1:/p2p-secret.key" + + # The ha-el-* overrides below only take effect when + # docker-compose-cluster.yml is passed as well; otherwise these services do + # not exist and compose ignores them. + ha-el-0: + <<: *reth-service + command: + - node + - --chain=/genesis.json + - --datadir=/db + - --http + - --http.addr=0.0.0.0 + - --http.port=8545 + - --http.api=web3,debug,eth,txpool,net,trace,admin,reth + - --ws + - --ws.addr=0.0.0.0 + - --ws.port=8546 + - --ws.api=web3,debug,eth,txpool,net,trace,admin,reth + - --authrpc.addr=0.0.0.0 + - --authrpc.port=8551 + - --authrpc.jwtsecret=/jwt-secret.txt + - --nat=none + - --disable-discovery + - --p2p-secret-key=/p2p-secret.key + - --trusted-peers=enode://73edbabab8727eba67dd5c2964c610decf28ec73e72d5bc395cc5be3c61ebd76cf72ada700b10926c3d2f9e86d64fafb92c6d02deb48c16439040b7c37f86c2d@ha-el-1:30303,enode://4a1a481e53b57ea351a08d945443cdff9cb7911e9121e8240efa99e4214582c0e383a30706349ce1be88b81d0f6344c401823a6624569e20d26880774279c41c@ha-el-2:30303,enode://58e698ea2dd8a76e0cb185d13c1faabf223b60c89fef988c8b89496571056d6c2922109537bb291cd87f2ec09a23ac37d59bde2c7a4885d07b7b641cadff2921@morph-el-0:30303,enode://bd755ce0bc8c06b4444b9013e8d1215a02e2b53f39f746f060c292ba2f6877d7b702374f006a49a7b1506bf1bc027b43824859d081283e6bac97c8600cdf3fee@morph-el-1:30303 + volumes: + - "${PWD}/ha-nodekey0:/p2p-secret.key" + + ha-el-1: + <<: *reth-service + command: + - node + - --chain=/genesis.json + - --datadir=/db + - --http + - --http.addr=0.0.0.0 + - --http.port=8545 + - --http.api=web3,debug,eth,txpool,net,trace,admin,reth + - --ws + - --ws.addr=0.0.0.0 + - --ws.port=8546 + - --ws.api=web3,debug,eth,txpool,net,trace,admin,reth + - --authrpc.addr=0.0.0.0 + - --authrpc.port=8551 + - --authrpc.jwtsecret=/jwt-secret.txt + - --nat=none + - --disable-discovery + - --p2p-secret-key=/p2p-secret.key + - --trusted-peers=enode://343bbe0507fd72946e8d57d5b42630dc905aed7e512945eb2dfd99a0d27bb1c7e84ba621c3408242d713a972a581374ea83a4d134651c373c4f2e8425ba99857@ha-el-0:30303,enode://4a1a481e53b57ea351a08d945443cdff9cb7911e9121e8240efa99e4214582c0e383a30706349ce1be88b81d0f6344c401823a6624569e20d26880774279c41c@ha-el-2:30303,enode://58e698ea2dd8a76e0cb185d13c1faabf223b60c89fef988c8b89496571056d6c2922109537bb291cd87f2ec09a23ac37d59bde2c7a4885d07b7b641cadff2921@morph-el-0:30303,enode://bd755ce0bc8c06b4444b9013e8d1215a02e2b53f39f746f060c292ba2f6877d7b702374f006a49a7b1506bf1bc027b43824859d081283e6bac97c8600cdf3fee@morph-el-1:30303 + volumes: + - "${PWD}/ha-nodekey1:/p2p-secret.key" + + ha-el-2: + <<: *reth-service + command: + - node + - --chain=/genesis.json + - --datadir=/db + - --http + - --http.addr=0.0.0.0 + - --http.port=8545 + - --http.api=web3,debug,eth,txpool,net,trace,admin,reth + - --ws + - --ws.addr=0.0.0.0 + - --ws.port=8546 + - --ws.api=web3,debug,eth,txpool,net,trace,admin,reth + - --authrpc.addr=0.0.0.0 + - --authrpc.port=8551 + - --authrpc.jwtsecret=/jwt-secret.txt + - --nat=none + - --disable-discovery + - --p2p-secret-key=/p2p-secret.key + - --trusted-peers=enode://343bbe0507fd72946e8d57d5b42630dc905aed7e512945eb2dfd99a0d27bb1c7e84ba621c3408242d713a972a581374ea83a4d134651c373c4f2e8425ba99857@ha-el-0:30303,enode://73edbabab8727eba67dd5c2964c610decf28ec73e72d5bc395cc5be3c61ebd76cf72ada700b10926c3d2f9e86d64fafb92c6d02deb48c16439040b7c37f86c2d@ha-el-1:30303,enode://58e698ea2dd8a76e0cb185d13c1faabf223b60c89fef988c8b89496571056d6c2922109537bb291cd87f2ec09a23ac37d59bde2c7a4885d07b7b641cadff2921@morph-el-0:30303,enode://bd755ce0bc8c06b4444b9013e8d1215a02e2b53f39f746f060c292ba2f6877d7b702374f006a49a7b1506bf1bc027b43824859d081283e6bac97c8600cdf3fee@morph-el-1:30303 + volumes: + - "${PWD}/ha-nodekey2:/p2p-secret.key" diff --git a/ops/docker/ha-nodekey0 b/ops/docker/ha-nodekey0 index b2235d998..3608fda3f 100644 --- a/ops/docker/ha-nodekey0 +++ b/ops/docker/ha-nodekey0 @@ -1 +1 @@ -543b2702353da96f244f35fb73a3495dd8a94b5628eafab0ca1bb85e9986d83c +543b2702353da96f244f35fb73a3495dd8a94b5628eafab0ca1bb85e9986d83c \ No newline at end of file diff --git a/ops/docker/ha-nodekey1 b/ops/docker/ha-nodekey1 index 75bbfcceb..986df5a90 100644 --- a/ops/docker/ha-nodekey1 +++ b/ops/docker/ha-nodekey1 @@ -1 +1 @@ -1ff441a6f43f26057e63fc92859d3fa36f49942e280c9a518ee11b42b56c1457 +1ff441a6f43f26057e63fc92859d3fa36f49942e280c9a518ee11b42b56c1457 \ No newline at end of file diff --git a/ops/docker/ha-nodekey2 b/ops/docker/ha-nodekey2 index 206c5a341..5bfcb857b 100644 --- a/ops/docker/ha-nodekey2 +++ b/ops/docker/ha-nodekey2 @@ -1 +1 @@ -5146627e8b60d6dca6666c4996083e5bd551fdbe56c6c4a7a53791274a574bd4 +5146627e8b60d6dca6666c4996083e5bd551fdbe56c6c4a7a53791274a574bd4 \ No newline at end of file diff --git a/ops/docker/nodekey2 b/ops/docker/nodekey2 index e8f14ad4a..90952052f 100644 --- a/ops/docker/nodekey2 +++ b/ops/docker/nodekey2 @@ -1 +1 @@ -b9b15801462f01ba61c1f6921c6214c0a4a87f010022086c1544b794fde6fc7e +b9b15801462f01ba61c1f6921c6214c0a4a87f010022086c1544b794fde6fc7e \ No newline at end of file diff --git a/ops/docker/static-nodes-cluster.json b/ops/docker/static-nodes-cluster.json new file mode 100644 index 000000000..34725bddb --- /dev/null +++ b/ops/docker/static-nodes-cluster.json @@ -0,0 +1,6 @@ +["enode://343bbe0507fd72946e8d57d5b42630dc905aed7e512945eb2dfd99a0d27bb1c7e84ba621c3408242d713a972a581374ea83a4d134651c373c4f2e8425ba99857@ha-el-0:30303", + "enode://73edbabab8727eba67dd5c2964c610decf28ec73e72d5bc395cc5be3c61ebd76cf72ada700b10926c3d2f9e86d64fafb92c6d02deb48c16439040b7c37f86c2d@ha-el-1:30303", + "enode://4a1a481e53b57ea351a08d945443cdff9cb7911e9121e8240efa99e4214582c0e383a30706349ce1be88b81d0f6344c401823a6624569e20d26880774279c41c@ha-el-2:30303", + "enode://58e698ea2dd8a76e0cb185d13c1faabf223b60c89fef988c8b89496571056d6c2922109537bb291cd87f2ec09a23ac37d59bde2c7a4885d07b7b641cadff2921@morph-el-0:30303", + "enode://bd755ce0bc8c06b4444b9013e8d1215a02e2b53f39f746f060c292ba2f6877d7b702374f006a49a7b1506bf1bc027b43824859d081283e6bac97c8600cdf3fee@morph-el-1:30303" +]