morph-l2 · panos-xyz · Sep 2, 2026 · Aug 31, 2026
@@ -161,7 +161,15 @@ export MORPH_RETH_RUSTFLAGS
 export MORPH_RETH_DOCKER_TARGET
 export MORPH_RETH_ENTRYPOINT
 DEVNET_COMPOSE_FILES := -f docker-compose-devnet.yml
-DEVNET_CLEAN_COMPOSE_FILES := -f docker-compose-devnet.yml -f docker-compose-reth.yml -f docker-compose-cluster.yml
+DEVNET_CLEAN_COMPOSE_FILES := -f docker-compose-devnet.yml -f docker-compose-cluster.yml -f docker-compose-reth.yml
+
+# The cluster topology is layered before the execution-client override, so that
+# docker-compose-reth.yml has the last word on the ha-el-* image, entrypoint and
+# command. Swapping these two leaves the cluster nodes running geth even when
+# reth was asked for, because later -f files win.
+ifneq ($(DEVNET_CLUSTER_ENABLED),)
+DEVNET_COMPOSE_FILES += -f docker-compose-cluster.yml
+endif
 
 ifeq ($(EXECUTION_CLIENT),geth)
 DEVNET_EXECUTION_DEPS := submodules
@@ -175,9 +183,6 @@ endif
 else
 $(error unsupported EXECUTION_CLIENT "$(EXECUTION_CLIENT)", expected "geth" or "reth")
 endif
-ifneq ($(DEVNET_CLUSTER_ENABLED),)
-DEVNET_COMPOSE_FILES += -f docker-compose-cluster.yml
-endif
 
 devnet-up: $(DEVNET_EXECUTION_DEPS) go-ubuntu-builder
 	python3 ops/devnet-morph/main.py --polyrepo-dir=. --execution-client=$(EXECUTION_CLIENT) \

@@ -84,10 +84,57 @@ looking anywhere else.
 | L1 | `9545` | `9546` | beacon `4000` |
 | `morph-el-0` | `8545` | `8546` | `node-0` → `26657` |
 | `morph-el-1` | `8645` | `8646` | — |
-| `ha-geth-0/1/2` | `9145` / `9245` / `9345` | `9146` / `9246` / `9346` | `27657` / `27757` / `27857` |
+| `ha-el-0/1/2` | `9145` / `9245` / `9345` | `9146` / `9246` / `9346` | `27657` / `27757` / `27857` |
 
 `ha-node` admin API: `9501` / `9601` / `9701`.
 
+The execution-layer services are named `el` rather than `geth` because either
+client can back them: `docker-compose-cluster.yml` defines them as geth, and
+`docker-compose-reth.yml` overrides them to reth. That override only works
+because the cluster file is layered *before* the reth file — later `-f` files
+win, so the reverse order silently leaves the cluster on geth.
+
+## Execution-layer peering
+
+Discovery is off everywhere (`--nodiscover` / `--disable-discovery`), so peers
+are configured explicitly and the topology is fixed:
+
+- geth reads `static-nodes.json` (mounted into `morph-el-1`) and, for the
+  cluster, `static-nodes-cluster.json` (mounted into all three `ha-el-*`).
+- reth ignores those files and takes `--trusted-peers` on the command line.
+
+Both clients derive their identity from the same `nodekey*` / `ha-nodekey*`
+files, so a node's enode is the same whichever client is running. reth needs
+`--p2p-secret-key` for this; without it, it invents a random identity per
+datadir and no peer list can be written in advance. The key files must not have
+a trailing newline — reth rejects those with `malformed or out-of-range secret
+key`, while geth tolerates them either way.
+
+`morph-el-0` and `morph-el-1` only know each other. The `ha-el-*` nodes dial
+both of those plus each other, which keeps `ha-el-*` names out of the
+non-cluster setup, where they would not resolve.
+
+## Consensus-layer peering
+
+`setup_nodes.py` writes `persistent_peers` for every tendermint home, deriving
+each node ID from the `node_key.json` that ends up installed — which is why the
+key files are copied before the peer list is built. Overwriting a
+`node_key.json` changes the node's identity, so a hardcoded ID silently goes
+stale.
+
+The list contains only the nodes that actually run tendermint: `node-0` and the
+three `ha-node-*`. `node-1` runs with
+`MORPH_NODE_DERIVATION_VERIFY_MODE=layer1` and never starts tendermint, and
+`node-2` has no compose service at all; listing either just produces endless
+reconnect and DNS failures.
+
+The `ha-node-*` reaching each other matters: the sequencer hand-over waits for
+the block pool to report caught up, and a pool whose only peers are unreachable
+never does. That is the silent-stall-at-height-0 failure described above.
+
+RPC is served on `0.0.0.0:26657` inside each container so the published ports in
+the table above are actually reachable from the host.
+
 Each geth serves metrics on `6060` inside its container, with
 `--metrics.expensive` on. Without that flag every counter behind
 `metrics.EnabledExpensive` stays zero, which blanks `chain/account/*` and

@@ -66,10 +66,14 @@
 def compose_file_args(execution_client, cluster=False):
     """Return docker-compose -f flags for the chosen L2 execution client."""
     args = ['-f', 'docker-compose-devnet.yml']
-    if execution_client == 'reth':
-        args.extend(['-f', 'docker-compose-reth.yml'])
+    # The cluster topology comes before the execution-client override so that
+    # docker-compose-reth.yml gets the last word on the ha-el-* image,
+    # entrypoint and command. Later -f files win, so reversing these two leaves
+    # the cluster nodes on geth even when reth was requested.
     if cluster:
         args.extend(['-f', 'docker-compose-cluster.yml'])
+    if execution_client == 'reth':
+        args.extend(['-f', 'docker-compose-reth.yml'])
     return args
 
 

@@ -1,9 +1,109 @@
+import base64
+import hashlib
+import json
 import os
 import shutil
 import subprocess
 import sys
 import re
 
+# Directories that hold a node's tendermint home, in the order their config
+# files are processed. The first entry is the genesis validator.
+NODE_DIRS = ("node0", "node1", "node2", "ha-node0", "ha-node1", "ha-node2")
+
+# Directory name -> compose service hostname, used to build peer addresses.
+SERVICE_HOSTNAMES = {
+    "node0": "node-0",
+    "node1": "node-1",
+    "node2": "node-2",
+    "ha-node0": "ha-node-0",
+    "ha-node1": "ha-node-1",
+    "ha-node2": "ha-node-2",
+}
+
+# Nodes that actually run tendermint, and so can be dialed as peers. node-1
+# runs with MORPH_NODE_DERIVATION_VERIFY_MODE=layer1 and never starts
+# tendermint; node-2 has no compose service at all. Listing either as a peer
+# only produces endless reconnect and DNS lookup failures.
+TENDERMINT_PEERS = ("node0", "ha-node0", "ha-node1", "ha-node2")
+
+
+def tendermint_node_id(node_key_path):
+    """Derive a tendermint node ID from a node_key.json file.
+
+    The ID is the hex encoding of the first 20 bytes of sha256(pubkey). An
+    ed25519 private key is stored as seed(32) || pubkey(32), so the public half
+    is the tail of the decoded value.
+
+    IDs must be derived from the key files that are actually in place, which is
+    why this runs after the key files have been copied: overwriting a
+    node_key.json changes the node's identity.
+    """
+    with open(node_key_path) as f:
+        priv_key = json.load(f)["priv_key"]["value"]
+    pubkey = base64.b64decode(priv_key)[32:]
+    return hashlib.sha256(pubkey).hexdigest()[:40]
+
+
+def copy_key_files(docker_dir, devnet_dir):
+    """Install the fixed node keys and the shared genesis into each node home.
+
+    Only node0 gets a genesis validator key. The others must not have one: a
+    node holding the sole genesis validator key gets block sync disabled and
+    never hands over to the sequencer routines.
+    """
+    print("Copying key files...")
+
+    for node in NODE_DIRS:
+        source_dir = os.path.join(docker_dir, node)
+        dest_dir = os.path.join(devnet_dir, node, "config")
+
+        if not os.path.isdir(dest_dir):
+            print(f"Error: Missing destination directory for {node}. Exiting.")
+            sys.exit(1)
+
+        if os.path.isdir(source_dir):
+            shutil.copyfile(os.path.join(source_dir, "node_key.json"), os.path.join(dest_dir, "node_key.json"))
+
+        if node == "node0" and os.path.isdir(source_dir):
+            shutil.copyfile(os.path.join(source_dir, "priv_validator_key.json"), os.path.join(dest_dir, "priv_validator_key.json"))
+        else:
+            priv_validator_key = os.path.join(dest_dir, "priv_validator_key.json")
+            priv_validator_state = os.path.join(devnet_dir, node, "data", "priv_validator_state.json")
+            for validator_file in (priv_validator_key, priv_validator_state):
+                if os.path.exists(validator_file):
+                    os.remove(validator_file)
+
+        # Copy and rename genesis file
+        shutil.copyfile(os.path.join(docker_dir, "tendermint-devnet-genesis.json"), os.path.join(dest_dir, "genesis.json"))
+
+        print(f"Files copied successfully for {node}.")
+
+    print("All key files have been copied successfully.")
+
+
+def build_persistent_peers(devnet_dir):
+    """Map each node directory to the peer list it should dial.
+
+    Every tendermint-running node is given all the others, so the HA nodes
+    reach each other rather than only node-0. Without that the HA block pool
+    never reports caught up, the sequencer hand-over never runs, and the
+    cluster silently stalls at height 0.
+    """
+    addresses = {}
+    for node in TENDERMINT_PEERS:
+        node_key = os.path.join(devnet_dir, node, "config", "node_key.json")
+        node_id = tendermint_node_id(node_key)
+        addresses[node] = f"{node_id}@{SERVICE_HOSTNAMES[node]}:26656"
+
+    peers = {}
+    for node in NODE_DIRS:
+        peers[node] = ",".join(
+            address for peer, address in addresses.items() if peer != node
+        )
+    return peers
+
+
 def setup_devnet_nodes():
     """
     Set up the devnet nodes, modify configuration files using toml library, and copy key files.
@@ -33,14 +133,7 @@ def setup_devnet_nodes():
     devnet_dir = os.path.join(docker_dir, ".devnet")
     if os.path.exists(devnet_dir):
         old_topology_paths = [os.path.join(devnet_dir, f"node{i}") for i in range(3, 6)]
-        expected_paths = [
-            os.path.join(devnet_dir, "node0"),
-            os.path.join(devnet_dir, "node1"),
-            os.path.join(devnet_dir, "node2"),
-            os.path.join(devnet_dir, "ha-node0"),
-            os.path.join(devnet_dir, "ha-node1"),
-            os.path.join(devnet_dir, "ha-node2"),
-        ]
+        expected_paths = [os.path.join(devnet_dir, node) for node in NODE_DIRS]
         if any(os.path.exists(path) for path in old_topology_paths) or any(
                 not os.path.exists(path) for path in expected_paths):
             print("Existing stale devnet detected. Regenerating single-sequencer config.")
@@ -73,20 +166,17 @@ def setup_devnet_nodes():
         if os.path.exists(generated_path):
             os.rename(generated_path, desired_path)
 
+    # Install the key files first: node IDs are derived from node_key.json, so
+    # the peer addresses below must be computed from the final keys.
+    copy_key_files(docker_dir, devnet_dir)
+
+    persistent_peers = build_persistent_peers(devnet_dir)
+
     # Modify config.toml files.
     print("Modifying config.toml files...")
-    config_files = [
-        os.path.join(devnet_dir, node, "config", "config.toml")
-        for node in ("node0", "node1", "node2", "ha-node0", "ha-node1", "ha-node2")
-    ]
-
-    persistent_peers_value = (
-        "93e27ea2306e158a8146d5f44caaab97496797d2@node-0:26656,"
-        "7f78b7d7a7e6bad4faf68d5731d437f4288d96d0@node-1:26656,"
-        "06c699be2f9aeb9f7ec79f508a95ff80576deb12@node-2:26656"
-    )
 
-    for i, config_file in enumerate(config_files):
+    for i, node in enumerate(NODE_DIRS):
+        config_file = os.path.join(devnet_dir, node, "config", "config.toml")
         if not os.path.isfile(config_file):
             print(f"Error: {config_file} not found. Exiting.")
             sys.exit(1)
@@ -102,7 +192,11 @@ def setup_devnet_nodes():
         content = content.replace('send_rate = 5120000', 'send_rate = 52428800')
         content = content.replace('recv_rate = 5120000', 'recv_rate = 102428800')
         content = content.replace('block_sync = false', 'block_sync = true')
-        content = re.sub(r'persistent_peers\s*=\s*".*?"', f'persistent_peers = "{persistent_peers_value}"', content)
+        content = re.sub(r'persistent_peers\s*=\s*".*?"', f'persistent_peers = "{persistent_peers[node]}"', content)
+
+        # Serve the RPC on all interfaces so the published container ports
+        # (26657, 27657, 27757, 27857) are reachable from the host.
+        content = content.replace('laddr = "tcp://127.0.0.1:26657"', 'laddr = "tcp://0.0.0.0:26657"')
 
         # Modify pex for the sequencer validator node.
         if i == 0:
@@ -115,35 +209,4 @@ def setup_devnet_nodes():
             f.write(content)
 
     print("All config.toml files have been updated successfully.")
-
-    # Copy key files to devnet node directories
-    print("Copying key files...")
-    node_dirs = ["node0", "node1", "node2", "ha-node0", "ha-node1", "ha-node2"]
-
-    for node in node_dirs:
-        source_dir = os.path.join(docker_dir, node)
-        dest_dir = os.path.join(devnet_dir, node, "config")
-
-        if not os.path.isdir(dest_dir):
-            print(f"Error: Missing destination directory for {node}. Exiting.")
-            sys.exit(1)
-
-        if os.path.isdir(source_dir):
-            shutil.copyfile(os.path.join(source_dir, "node_key.json"), os.path.join(dest_dir, "node_key.json"))
-
-        if node == "node0" and os.path.isdir(source_dir):
-            shutil.copyfile(os.path.join(source_dir, "priv_validator_key.json"), os.path.join(dest_dir, "priv_validator_key.json"))
-        else:
-            priv_validator_key = os.path.join(dest_dir, "priv_validator_key.json")
-            priv_validator_state = os.path.join(devnet_dir, node, "data", "priv_validator_state.json")
-            for validator_file in (priv_validator_key, priv_validator_state):
-                if os.path.exists(validator_file):
-                    os.remove(validator_file)
-
-        # Copy and rename genesis file
-        shutil.copyfile(os.path.join(docker_dir, "tendermint-devnet-genesis.json"), os.path.join(dest_dir, "genesis.json"))
-
-        print(f"Files copied successfully for {node}.")
-
-    print("All key files have been copied successfully.")
     print("Devnet nodes setup completed successfully.")