From 135e8e187226c08cfcd48f166df0a03dbafb3db4 Mon Sep 17 00:00:00 2001 From: cc Date: Mon, 7 Sep 2026 23:55:40 -0700 Subject: [PATCH] fix: unify Nix package preparation for installed native modules --- .github/scripts/check_sdist.py | 2 + .github/scripts/native_wheel_smoke.py | 97 +++++++++++ .github/scripts/wheel_smoke.py | 10 ++ .github/workflows/release-build-check.yml | 4 + .github/workflows/release.yml | 1 + bin/build-native-modules | 25 ++- dimos/cli/can.py | 3 +- dimos/cli/commands/native.py | 31 ++++ dimos/cli/dimos.py | 2 + dimos/cli/hardware/g1.py | 8 +- dimos/cli/test_can.py | 5 + dimos/core/native_module.py | 28 ++- dimos/core/native_package.py | 138 +++++++++++++++ dimos/core/test_build_native_modules.py | 29 +++- dimos/core/test_native_package.py | 159 ++++++++++++++++++ .../experimental/memory/rust_cli_recorder.py | 56 ++---- dimos/experimental/memory/rust_recorder.py | 20 +-- dimos/experimental/memory/rust_types.py | 26 +++ .../memory/test_rust_cli_recorder.py | 103 +++++++----- .../experimental/memory/test_rust_recorder.py | 10 +- .../memory/test_rust_recorder_e2e.py | 19 +-- .../robot/bosdyn/spot/recorder.py | 3 +- .../hardware/sensors/lidar/fastlio2/module.py | 4 +- .../sensors/lidar/fastlio2/recorder.py | 3 +- dimos/hardware/sensors/lidar/livox/module.py | 4 +- .../hardware/sensors/lidar/pointlio/module.py | 4 +- .../sensors/lidar/pointlio/recorder.py | 3 +- dimos/memory/module.py | 9 +- dimos/memory/store/mcap.py | 27 ++- dimos/memory/store/test_mcap.py | 10 +- dimos/memory/type/recording.py | 24 +++ dimos/native_packages.json | 22 +++ docs/adr/0001-native-package-preparation.md | 15 ++ docs/usage/native_modules.md | 81 ++++++--- docs/usage/recording.md | 16 +- pyproject.toml | 6 +- setup.py | 36 +++- uv.lock | 6 +- 38 files changed, 850 insertions(+), 199 deletions(-) create mode 100644 .github/scripts/native_wheel_smoke.py create mode 100644 dimos/cli/commands/native.py create mode 100644 dimos/core/native_package.py create mode 100644 dimos/core/test_native_package.py create mode 100644 dimos/experimental/memory/rust_types.py create mode 100644 dimos/memory/type/recording.py create mode 100644 dimos/native_packages.json create mode 100644 docs/adr/0001-native-package-preparation.md diff --git a/.github/scripts/check_sdist.py b/.github/scripts/check_sdist.py index 248e62baae..291114332e 100644 --- a/.github/scripts/check_sdist.py +++ b/.github/scripts/check_sdist.py @@ -27,6 +27,8 @@ # Paths below the dimos-/ root that every release sdist must carry. REQUIRED = ( + "dimos/native_packages.json", + "dimos/_native_revision.json", "web/cockpit/dist/index.html", "web/deno.lock", "web/relay/main.ts", diff --git a/.github/scripts/native_wheel_smoke.py b/.github/scripts/native_wheel_smoke.py new file mode 100644 index 0000000000..a2b91c4028 --- /dev/null +++ b/.github/scripts/native_wheel_smoke.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Record and read native artifacts from a core-only wheel installation. + +Run with the installed environment's Python, from outside the checkout. Nix +is required; preparation may download or build the pinned native package. +""" + +from dataclasses import replace +from pathlib import Path +import socket +import subprocess +import sys +import tempfile +import time +import uuid + +from dimos.constants import DIMOS_PROJECT_ROOT +from dimos.core.global_config import global_config +from dimos.core.native_package import ensure_native_package, source_revision +from dimos.core.transport import LCMTransport +from dimos.experimental.memory.rust_cli_recorder import RustRecordingSession, make_plan +from dimos.msgs.geometry_msgs.Vector3 import Vector3 +from dimos.msgs.sensor_msgs.Imu import Imu + + +def main() -> None: + assert not (DIMOS_PROJECT_ROOT / ".git").exists(), "must import the installed wheel" + assert not (DIMOS_PROJECT_ROOT / "dimos/experimental/memory/rust").exists() + executable = ensure_native_package("dimos-memory-recorder") + assert executable.is_file() + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + url = f"udpm://239.255.76.67:{port}?ttl=0" + transport = LCMTransport(f"/native-wheel-{uuid.uuid4().hex}", Imu, url=url) + with tempfile.TemporaryDirectory(prefix="dimos-native-recording-") as directory: + try: + transport.start() + for kind, suffix in (("sqlite", "db"), ("mcap", "mcap")): + global_config.update( + record_engine="rust", record=kind, record_topics="*", replay=False + ) + plan = make_plan({("imu", Imu): transport}) + # Artifact location is explicit; executable preparation stays unmodified. + plan = replace(plan, path=Path(directory) / f"memory.{suffix}") + session = RustRecordingSession(plan) + try: + session.start() + for _ in range(3): + transport.broadcast( + None, Imu(ts=22.5, frame_id="imu", angular_velocity=Vector3(1, 2, 3)) + ) + time.sleep(0.05) + finally: + session.stop() + result = subprocess.run( + [sys.executable, "-m", "dimos.cli.dimos", "mem", "summary", str(plan.path)], + check=True, + capture_output=True, + text=True, + ) + assert "imu" in result.stdout, result.stdout + if kind == "mcap": + subprocess.run( + [ + sys.executable, + "-m", + "dimos.cli.dimos", + "mem", + "rerun", + str(plan.path), + "--no-gui", + ], + check=True, + ) + assert plan.path.with_suffix(".rrd").is_file() + finally: + transport.stop() + print(f"Native wheel smoke passed: {source_revision()} -> {executable}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/wheel_smoke.py b/.github/scripts/wheel_smoke.py index 981efc19d4..eecc005617 100644 --- a/.github/scripts/wheel_smoke.py +++ b/.github/scripts/wheel_smoke.py @@ -23,8 +23,11 @@ import json from pathlib import Path +import subprocess +import sys import urllib.request +from dimos.core.native_package import native_packages, package_reference, source_revision from dimos.navigation.replanning_a_star.min_cost_astar_ext import min_cost_astar_cpp # noqa: F401 from dimos.web.relay_bridge import locate from dimos.web.relay_bridge.relay_process import RelayProcess @@ -44,6 +47,13 @@ def main() -> None: + for command in ("native", "mem"): + subprocess.run([sys.executable, "-m", "dimos.cli.dimos", command, "--help"], check=True) + revision = source_revision() + for package in native_packages().values(): + reference = package_reference(package) + if not reference.startswith(f"github:dimensionalOS/dimos/{revision}?"): + raise SystemExit(f"installed native package resolved a checkout: {reference}") dist = Path(locate.__file__).resolve().parent / "_relay_dist" for rel in REQUIRED: if not (dist / rel).is_file(): diff --git a/.github/workflows/release-build-check.yml b/.github/workflows/release-build-check.yml index 60e33bdd77..669de19898 100644 --- a/.github/workflows/release-build-check.yml +++ b/.github/workflows/release-build-check.yml @@ -20,6 +20,8 @@ on: - web/deno.json - web/deno.lock - dimos/utils/deno.py + - dimos/native_packages.json + - dimos/core/native_package.py permissions: {} @@ -39,6 +41,8 @@ jobs: uses: ./.github/actions/build-cockpit - name: Build one wheel uses: pypa/cibuildwheel@v4.2.0 + env: + CIBW_ENVIRONMENT: DIMOS_BUILD_REVISION=${{ github.sha }} with: only: cp312-manylinux_x86_64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 51cb70d6b4..65e0fc5e20 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -116,6 +116,7 @@ jobs: attestations: write # For uploading the SLSA build-provenance attestation env: CIBW_ARCHS: ${{ matrix.arch }} + CIBW_ENVIRONMENT: DIMOS_BUILD_REVISION=${{ github.sha }} # Skip musllinux (Alpine) — Pinocchio's build ecosystem (pin/coal/cmeel-*) # is glibc-only; cmeel-assimp >= 6.0.5 has no musllinux wheels. CIBW_SKIP: "*-musllinux_*" diff --git a/bin/build-native-modules b/bin/build-native-modules index 2dff0a285d..e847118fad 100755 --- a/bin/build-native-modules +++ b/bin/build-native-modules @@ -45,6 +45,7 @@ import ast from concurrent.futures import ProcessPoolExecutor, as_completed from dataclasses import dataclass import hashlib +import json import os from pathlib import Path, PurePosixPath import re @@ -54,7 +55,7 @@ import time import urllib.error import urllib.request -MARKER_SALT = 1 +MARKER_SALT = 2 REPO_ROOT = Path(__file__).resolve().parent.parent PACKAGE_DIR = REPO_ROOT / "dimos" @@ -137,7 +138,9 @@ def is_nix_build(command: str | None) -> bool: def discover() -> tuple[DiscoveredModule, ...]: + catalog = json.loads((PACKAGE_DIR / "native_packages.json").read_text()) found = [] + packages_seen = set() provisioned_seen = set() for dirpath, dirnames, filenames in os.walk(PACKAGE_DIR): dirnames[:] = sorted(d for d in dirnames if d not in IGNORED_DIRS) @@ -159,6 +162,21 @@ def discover() -> tuple[DiscoveredModule, ...]: provisioned_seen.add(qualname) continue build_command = _literal_default(node, "build_command", source) + package_id = _literal_default(node, "native_package", source) + if package_id: + if package_id not in catalog: + raise SystemExit(f"{source}: unknown native_package {package_id!r}") + package = catalog[package_id] + packages_seen.add(package_id) + found.append( + DiscoveredModule( + qualname=qualname, + source=source, + build_command=f"nix build -L .#{package['attribute']}", + build_dir=package["flake_dir"], + ) + ) + continue if not is_nix_build(build_command): continue cwd = _literal_default(node, "cwd", source) @@ -180,6 +198,10 @@ def discover() -> tuple[DiscoveredModule, ...]: f"EXTERNALLY_PROVISIONED entries not found in the tree: {sorted(stale)}" " — remove them from bin/build-native-modules" ) + if packages_seen != set(catalog): + raise SystemExit( + f"Native packages without a module: {sorted(set(catalog) - packages_seen)}" + ) return tuple(sorted(found)) @@ -377,6 +399,7 @@ def build_manifest(modules: tuple[DiscoveredModule, ...]) -> str: f"salt {MARKER_SALT}", f"script {_git_object_hash('bin/build-native-modules')}", f"cachix {os.environ.get('CACHIX_CACHE_NAME', '')}", + f"catalog {(PACKAGE_DIR / 'native_packages.json').read_text().strip()}", *(f"module {m.qualname} dir={m.build_dir} cmd={m.build_command}" for m in modules), *(f"path {path} {_git_object_hash(path)}" for path in top), ] diff --git a/dimos/cli/can.py b/dimos/cli/can.py index c7f142be64..bb219b4d2b 100644 --- a/dimos/cli/can.py +++ b/dimos/cli/can.py @@ -16,12 +16,12 @@ from __future__ import annotations +import importlib import os import shlex import subprocess import sys -import can_motor_control import typer app = typer.Typer(help="Discover and configure CAN interfaces", no_args_is_help=True) @@ -64,6 +64,7 @@ def list_devices() -> None: typer.echo(output or "No SocketCAN interfaces found") return if sys.platform == "darwin": + can_motor_control = importlib.import_module("can_motor_control") try: devices = can_motor_control.list_gs_usb_devices( vendor_id=GS_USB_VENDOR_ID, diff --git a/dimos/cli/commands/native.py b/dimos/cli/commands/native.py new file mode 100644 index 0000000000..952c7dfe78 --- /dev/null +++ b/dimos/cli/commands/native.py @@ -0,0 +1,31 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prepare native executables without starting a robot.""" + +import typer + +from dimos.core.native_package import ensure_native_package + +native_app = typer.Typer(help="Prepare native packages through Nix and Cachix.") + + +@native_app.command() +def prepare(package_id: str) -> None: + """Download or build PACKAGE_ID using the same inputs as automatic startup.""" + try: + executable = ensure_native_package(package_id) + except (ValueError, RuntimeError) as error: + raise typer.BadParameter(str(error)) from error + typer.echo(str(executable)) diff --git a/dimos/cli/dimos.py b/dimos/cli/dimos.py index 0a8565986f..d4eb08219c 100644 --- a/dimos/cli/dimos.py +++ b/dimos/cli/dimos.py @@ -61,6 +61,7 @@ from dimos.cli.commands.lifecycle import log_cmd, restart, run, status, stop from dimos.cli.commands.map import map_app from dimos.cli.commands.mcp import agent_send_cmd, mcp_app +from dimos.cli.commands.native import native_app from dimos.cli.commands.rerun_bridge import rerun_bridge_cmd from dimos.cli.commands.topic import topic_app from dimos.cli.commands.tuis import agentspy, humancli, lcmspy, spy, top @@ -146,6 +147,7 @@ def cli_main() -> None: from dimos.memory.cli.app import mem_app main.add_typer(mem_app, name="mem") +main.add_typer(native_app, name="native") from dimos.evals.cli import app as evals_app diff --git a/dimos/cli/hardware/g1.py b/dimos/cli/hardware/g1.py index db2d54bc58..aa0e140035 100644 --- a/dimos/cli/hardware/g1.py +++ b/dimos/cli/hardware/g1.py @@ -16,6 +16,7 @@ from __future__ import annotations +import importlib import time from typing import Any, NoReturn, Protocol, TypeGuard @@ -25,7 +26,6 @@ from dimos.msgs.sensor_msgs.JointState import JointState from dimos.porcelain.dimos import Dimos from dimos.porcelain.module_handle import ModuleHandle -from dimos.robot.unitree.g1.manip_config import G1_READY_JOINTS, G1_READY_SPEED_SCALE app = typer.Typer(help="Operate a running Unitree G1 stack safely") @@ -162,10 +162,12 @@ def _execute_ready_pose( ) -> None: _require_armed_and_enabled(coordinator) _require_teleop_disengaged(coordinator) + manip_config = importlib.import_module("dimos.robot.unitree.g1.manip_config") targets = { - group: JointState(position=list(positions)) for group, positions in G1_READY_JOINTS.items() + group: JointState(position=list(positions)) + for group, positions in manip_config.G1_READY_JOINTS.items() } - planned = manipulation.plan_to_joints(targets, speed_scale=G1_READY_SPEED_SCALE) + planned = manipulation.plan_to_joints(targets, speed_scale=manip_config.G1_READY_SPEED_SCALE) if not planned.succeeded: _abort(f"ready-pose planning failed: {planned}") executed = manipulation.execute(blocking=True) diff --git a/dimos/cli/test_can.py b/dimos/cli/test_can.py index 55538ea479..74ab1adce4 100644 --- a/dimos/cli/test_can.py +++ b/dimos/cli/test_can.py @@ -24,6 +24,11 @@ from dimos.cli.dimos import main +@pytest.fixture(autouse=True) +def can_backend(mocker: MockerFixture) -> None: + mocker.patch.dict(sys.modules, {"can_motor_control": Mock(TransportError=RuntimeError)}) + + def _subprocess_argv(run: Mock) -> list[list[str]]: return [call.args[0] for call in run.call_args_list] diff --git a/dimos/core/native_module.py b/dimos/core/native_module.py index 84fea6b685..4caba152bb 100644 --- a/dimos/core/native_module.py +++ b/dimos/core/native_module.py @@ -60,6 +60,7 @@ class MyCppModule(NativeModule): from dimos.core.core import rpc from dimos.core.global_config import global_config from dimos.core.module import Module, ModuleConfig +from dimos.core.native_package import ensure_native_package from dimos.core.transport_factory import session_config from dimos.protocol.service.spec import SessionConfig from dimos.utils.logging_config import setup_logger @@ -117,7 +118,8 @@ class LogFormat(enum.Enum): class NativeModuleConfig(ModuleConfig): """Configuration for a native subprocess module.""" - executable: str + executable: str = "" + native_package: str | None = None build_command: str | None = None cwd: str | None = None extra_args: list[str] = Field(default_factory=list) @@ -139,7 +141,13 @@ class NativeModuleConfig(ModuleConfig): base_fields: frozenset[str] = frozenset() @model_validator(mode="after") - def _session_needs_the_stdin_line(self) -> NativeModuleConfig: + def _validate_native_config(self) -> NativeModuleConfig: + if not self.executable and not self.native_package: + raise ValueError("A native module requires an executable or native_package") + if self.native_package and (self.executable or self.build_command or self.cwd): + raise ValueError( + "native_package cannot be combined with executable, build_command, or cwd" + ) if self.session is not None and not self.stdin_config: raise ValueError( f"{self.executable} pins a session config but has stdin_config off, " @@ -211,10 +219,15 @@ class NativeModule(Module): _watchdog: threading.Thread | None = None _stopping: bool = False _stop_lock: threading.Lock + _prepared_executable: str | None = None + + @property + def _executable(self) -> str: + return self._prepared_executable or self.config.executable @functools.cached_property def _module_label(self) -> str: - exe = Path(self.config.executable).name if self.config.executable else "?" + exe = Path(self._executable).name if self._executable else self.config.native_package return f"{type(self).__name__}({exe})" def __init__(self, **kwargs: Any) -> None: @@ -224,7 +237,7 @@ def __init__(self, **kwargs: Any) -> None: if self.config.cwd is not None and not Path(self.config.cwd).is_absolute(): base_dir = Path(inspect.getfile(type(self))).resolve().parent self.config.cwd = str(base_dir / self.config.cwd) - if not Path(self.config.executable).is_absolute(): + if self.config.executable and not Path(self.config.executable).is_absolute(): # The spawn runs from the executable's own directory, so a relative # path has to be resolved before then or it resolves against itself. base = Path(self.config.cwd) if self.config.cwd is not None else Path.cwd() @@ -260,7 +273,7 @@ def _session(self) -> SessionConfig: def _argv(self, topics: dict[str, str]) -> list[str]: """The command line the native process is spawned with.""" - cmd = [self.config.executable] + cmd = [self._executable] for name, topic_str in topics.items(): cmd.extend([f"--{name}", topic_str]) cmd.extend(self.config.to_cli_args()) @@ -299,7 +312,7 @@ def start(self) -> None: stdin_blob = self._stdin_blob(topics) if self.config.stdin_config else None env = self._spawn_env() - cwd = self.config.cwd or str(Path(self.config.executable).resolve().parent) + cwd = self.config.cwd or str(Path(self._executable).resolve().parent) logger.info( "Starting native process", @@ -461,6 +474,9 @@ def _read_log_stream( stream.close() def _maybe_build(self) -> None: + if self.config.native_package: + self._prepared_executable = str(ensure_native_package(self.config.native_package)) + return exe = Path(self.config.executable) if self.config.build_command is None: diff --git a/dimos/core/native_package.py b/dimos/core/native_package.py new file mode 100644 index 0000000000..d128c3153a --- /dev/null +++ b/dimos/core/native_package.py @@ -0,0 +1,138 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prepare pinned Nix packages for installed DimOS and local checkouts.""" + +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re +import shutil +import subprocess + +from filelock import FileLock + +from dimos.constants import CACHE_DIR, DIMOS_PROJECT_ROOT +from dimos.utils.cache import cache_usage_guard +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() +_PACKAGE_DIR = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class NativePackage: + flake_dir: str + attribute: str + executable: str + + +def native_packages() -> dict[str, NativePackage]: + """Load the catalog also consumed by the dependency-free CI publisher.""" + data = json.loads((_PACKAGE_DIR / "native_packages.json").read_text()) + return {name: NativePackage(**spec) for name, spec in data.items()} + + +def source_revision() -> str: + """The immutable source revision carried by wheels and source distributions.""" + metadata = _PACKAGE_DIR / "_native_revision.json" + if not metadata.is_file(): + raise RuntimeError("DimOS distribution is missing native source revision metadata") + revision = json.loads(metadata.read_text()).get("revision", "") + if not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}", revision): + raise RuntimeError("DimOS distribution has invalid native source revision metadata") + return revision + + +def package_reference(package: NativePackage) -> str: + """Use local Git inputs only when the imported package belongs to a checkout.""" + if (DIMOS_PROJECT_ROOT / ".git").exists(): + flake = DIMOS_PROJECT_ROOT / package.flake_dir + return f"{flake}#{package.attribute}" + return ( + f"github:dimensionalOS/dimos/{source_revision()}" + f"?dir={package.flake_dir}#{package.attribute}" + ) + + +def _nix(arguments: list[str]) -> str: + command = [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "--extra-substituters", + "https://dimensionalos.cachix.org", + "--extra-trusted-public-keys", + "dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI=", + *arguments, + ] + # Stdout is machine-readable; stream stderr so evaluation and builds remain visible. + with subprocess.Popen(command, stdout=subprocess.PIPE, text=True) as process: + assert process.stdout is not None + output = process.stdout.read() + status = process.wait() + if status: + raise RuntimeError( + f"Native package preparation failed (exit {status}): {' '.join(command)}" + ) + return output.strip() + + +def ensure_native_package(package_id: str) -> Path: + """Reuse, substitute, or build a package and return its immutable executable path. + + Evaluate on every preparation so edits to native inputs cannot reuse a stale + checkout result. Nix owns input hashing and substitution; no Git dirty-bit + heuristic or Python build cache is involved. + """ + packages = native_packages() + if package_id not in packages: + raise ValueError( + f"Unknown native package {package_id!r}; choose from {', '.join(packages)}" + ) + if shutil.which("nix") is None: + raise RuntimeError( + "Native packages require Nix with flakes enabled. " + "See https://github.com/dimensionalOS/dimos/blob/main/docs/installation/nix.md and configure the dimensionalos Cachix cache." + ) + package = packages[package_id] + reference = package_reference(package) + logger.info("Preparing native package", package=package_id, source=reference) + with cache_usage_guard(): + derivation = _nix(["eval", "--raw", "--no-update-lock-file", f"{reference}.drvPath"]) + if not derivation.startswith("/nix/store/") or not derivation.endswith(".drv"): + raise RuntimeError(f"Nix returned an invalid derivation path: {derivation!r}") + directory = CACHE_DIR / "native" / Path(derivation).name + directory.mkdir(parents=True, exist_ok=True) + with FileLock(directory / "prepare.lock"): + result = directory / "result" + executable = result / "bin" / package.executable + if not executable.is_file(): + _nix( + [ + "build", + "-L", + f"{derivation}^out", + "--out-link", + str(result), + "--max-jobs", + "1", + "--cores", + "2", + ] + ) + if not executable.is_file() or not os.access(executable, os.X_OK): + raise RuntimeError(f"Native package {package_id!r} did not provide {executable}") + return executable.resolve() diff --git a/dimos/core/test_build_native_modules.py b/dimos/core/test_build_native_modules.py index 1cf0f44837..8e9d39321a 100644 --- a/dimos/core/test_build_native_modules.py +++ b/dimos/core/test_build_native_modules.py @@ -27,7 +27,6 @@ import importlib from importlib.machinery import SourceFileLoader from importlib.util import module_from_spec, spec_from_loader -import inspect import json import os from pathlib import Path @@ -38,6 +37,7 @@ import pytest from dimos.constants import DIMOS_PROJECT_ROOT +from dimos.core.native_package import native_packages _SCRIPT_PATH = DIMOS_PROJECT_ROOT / "bin" / "build-native-modules" if not _SCRIPT_PATH.is_file(): @@ -70,6 +70,7 @@ class _ClassDef(NamedTuple): bases: tuple[str, ...] command: str | None # build_command literal defined in this class body command_kind: str # "absent" | "literal" | "opaque" + package_id: str | None def _base_names(node: ast.ClassDef) -> tuple[str, ...]: @@ -118,7 +119,20 @@ def _scan_all_config_classes() -> list[_ClassDef]: for node in ast.walk(ast.parse(path.read_text(), filename=rel)): if isinstance(node, ast.ClassDef): kind, command = _own_build_command(node) - classes.append(_ClassDef(rel, node.name, _base_names(node), command, kind)) + package_id = next( + ( + stmt.value.value + for stmt in node.body + if isinstance(stmt, ast.AnnAssign) + and isinstance(stmt.target, ast.Name) + and stmt.target.id == "native_package" + and isinstance(stmt.value, ast.Constant) + ), + None, + ) + classes.append( + _ClassDef(rel, node.name, _base_names(node), command, kind, package_id) + ) return classes @@ -169,7 +183,7 @@ def effective_command(cls: _ClassDef, seen: frozenset[str]) -> tuple[str, str | f"{cls.file}: {cls.name}.build_command must default to a plain string literal " "so bin/build-native-modules can read it without importing dimos" ) - if _SCRIPT.is_nix_build(command): + if _SCRIPT.is_nix_build(command) or cls.package_id: nix_configs.add((cls.file, cls.name)) return nix_configs @@ -198,11 +212,10 @@ def test_ast_extraction_matches_runtime() -> None: dotted, class_name = module.qualname.rsplit(".", 1) config_class = getattr(importlib.import_module(dotted), class_name) fields = config_class.model_fields - assert fields["build_command"].default == module.build_command - cwd = fields["cwd"].default - base_dir = Path(inspect.getfile(config_class)).resolve().parent - runtime_dir = Path(os.path.normpath(base_dir if cwd is None else base_dir / cwd)) - assert runtime_dir == (DIMOS_PROJECT_ROOT / module.build_dir).resolve() + package = native_packages()[fields["native_package"].default] + assert module.build_command == f"nix build -L .#{package.attribute}" + assert module.build_dir == package.flake_dir + assert fields["cwd"].default is None @pytest.mark.skipif(not _IN_GIT_CHECKOUT, reason="needs git HEAD for object hashes") diff --git a/dimos/core/test_native_package.py b/dimos/core/test_native_package.py new file mode 100644 index 0000000000..133faa4043 --- /dev/null +++ b/dimos/core/test_native_package.py @@ -0,0 +1,159 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native preparation must work without Python-adjacent build directories.""" + +from contextlib import nullcontext +from io import StringIO +import json +from pathlib import Path +import subprocess +from unittest.mock import MagicMock + +import pytest + +from dimos.core import native_package +from dimos.core.native_module import NativeModuleConfig +from dimos.experimental.memory.rust_recorder import RustRecorder +from dimos.hardware.sensors.lidar.fastlio2.module import FastLio2 +from dimos.hardware.sensors.lidar.livox.module import Mid360 +from dimos.hardware.sensors.lidar.pointlio.module import PointLio + + +@pytest.fixture +def isolated_packages(tmp_path, monkeypatch): + monkeypatch.setattr(native_package, "CACHE_DIR", tmp_path / "cache") + monkeypatch.setattr(native_package, "DIMOS_PROJECT_ROOT", tmp_path / "installed") + package_dir = tmp_path / "installed" / "dimos" + package_dir.mkdir(parents=True) + catalog = native_package._PACKAGE_DIR / "native_packages.json" + (package_dir / "native_packages.json").write_text(catalog.read_text()) + (package_dir / "_native_revision.json").write_text(json.dumps({"revision": "a" * 40})) + monkeypatch.setattr(native_package, "_PACKAGE_DIR", package_dir) + monkeypatch.setattr(native_package, "cache_usage_guard", nullcontext) + return package_dir + + +def test_installed_package_uses_pinned_source(isolated_packages): + package = native_package.native_packages()["dimos-memory-recorder"] + assert native_package.package_reference(package) == ( + "github:dimensionalOS/dimos/" + + "a" * 40 + + "?dir=dimos/experimental/memory/rust#dimos-memory-recorder" + ) + + +def test_checkout_uses_local_inputs(isolated_packages): + (isolated_packages.parent / ".git").write_text("gitdir: /unused") + package = native_package.native_packages()["dimos-memory-recorder"] + assert native_package.package_reference(package) == ( + f"{isolated_packages.parent}/dimos/experimental/memory/rust#dimos-memory-recorder" + ) + + +@pytest.mark.parametrize("metadata", [None, {"revision": "main"}, {"revision": 42}]) +def test_installed_package_never_guesses_revision(isolated_packages, metadata): + path = isolated_packages / "_native_revision.json" + if metadata is None: + path.unlink() + else: + path.write_text(json.dumps(metadata)) + with pytest.raises(RuntimeError, match="revision metadata"): + native_package.source_revision() + + +def test_preparation_reuses_exact_outputs_and_rechecks_inputs(isolated_packages, tmp_path, mocker): + mocker.patch.object(native_package.shutil, "which", return_value="/bin/nix") + generation = ["first"] + builds = [] + + def nix(arguments): + if arguments[0] == "eval": + return f"/nix/store/{generation[0]}.drv" + builds.append(arguments) + result = Path(arguments[arguments.index("--out-link") + 1]) + output = tmp_path / generation[0] + executable = output / "bin" / "dimos-memory-recorder" + executable.parent.mkdir(parents=True) + executable.write_text("#!/bin/sh\nexit 0\n") + executable.chmod(0o755) + result.symlink_to(output, target_is_directory=True) + return "" + + mocker.patch.object(native_package, "_nix", side_effect=nix) + first = native_package.ensure_native_package("dimos-memory-recorder") + assert native_package.ensure_native_package("dimos-memory-recorder") == first + assert len(builds) == 1 + generation[0] = "changed" + second = native_package.ensure_native_package("dimos-memory-recorder") + assert second != first + assert first.is_file() # Other worktrees/runs retain their rooted result. + assert len(builds) == 2 + assert builds[0][-4:] == ["--max-jobs", "1", "--cores", "2"] + + +def test_missing_nix_has_installation_guidance(isolated_packages, mocker): + mocker.patch.object(native_package.shutil, "which", return_value=None) + with pytest.raises(RuntimeError, match="require Nix"): + native_package.ensure_native_package("mid360") + + +def test_failed_evaluation_does_not_attempt_build(isolated_packages, mocker): + mocker.patch.object(native_package.shutil, "which", return_value="nix") + nix = mocker.patch.object(native_package, "_nix", side_effect=RuntimeError("download failed")) + with pytest.raises(RuntimeError, match="download failed"): + native_package.ensure_native_package("pointlio") + assert nix.call_count == 1 + + +def test_failed_nix_command_reports_status(mocker): + process = MagicMock(spec=subprocess.Popen) + process.__enter__.return_value = process + process.stdout = StringIO("") + process.wait.return_value = 7 + mocker.patch.object(native_package.subprocess, "Popen", return_value=process) + with pytest.raises(RuntimeError, match="exit 7"): + native_package._nix(["build", "example"]) + + +@pytest.mark.parametrize( + "module_class, package_id", + [ + (RustRecorder, "dimos-memory-recorder"), + (Mid360, "mid360"), + (PointLio, "pointlio"), + (FastLio2, "fastlio2"), + ], +) +def test_modules_use_prepared_executable_without_source_cwd( + module_class, package_id, tmp_path, mocker +): + executable = tmp_path / "store" / "bin" / "native" + prepare = mocker.patch( + "dimos.core.native_module.ensure_native_package", return_value=executable + ) + module = module_class() + try: + module._maybe_build() + assert module._argv({})[0] == str(executable) + assert module.config.cwd is None + assert "native_package" not in module.config.to_config_dict() + prepare.assert_called_once_with(package_id) + finally: + module.stop() + + +def test_package_does_not_accept_checkout_build_configuration(): + with pytest.raises(ValueError, match="cannot be combined"): + NativeModuleConfig(native_package="mid360", cwd="cpp") diff --git a/dimos/experimental/memory/rust_cli_recorder.py b/dimos/experimental/memory/rust_cli_recorder.py index cbb50ccb4a..b4e5156106 100644 --- a/dimos/experimental/memory/rust_cli_recorder.py +++ b/dimos/experimental/memory/rust_cli_recorder.py @@ -23,14 +23,14 @@ import signal import subprocess import threading -import time from typing import IO, Any from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT from dimos.core.global_config import global_config +from dimos.core.native_package import ensure_native_package from dimos.core.transport import LCMTransport, ZenohTransport from dimos.core.transport_factory import session_config -from dimos.experimental.memory.rust_recorder import RustStreamSpec +from dimos.experimental.memory.rust_types import RustStreamSpec from dimos.memory.store.sqlite import SqliteStore from dimos.memory.tap import matching, recording_dir from dimos.msgs.sensor_msgs.Image import Image @@ -38,9 +38,6 @@ logger = setup_logger() -_RUST_DIR = Path(__file__).resolve().parent / "rust" -_EXECUTABLE = _RUST_DIR / "result/bin/dimos-memory-recorder" -_BUILD_COMMAND = ("nix", "build", "-L", ".#dimos-memory-recorder") _READY_TIMEOUT = 10.0 _DEFAULT_ENCODING_THREADS = 4 _READY_MESSAGE = "memory recorder ready" @@ -55,36 +52,7 @@ class RustRecordingPlan: streams: list[RustStreamSpec] payload_types: dict[str, type[Any]] path: Path - - -def prepare_rust_recorder() -> None: - """Build the native recorder when it is unavailable or explicitly requested.""" - if global_config.record_engine != "rust" or not global_config.record or global_config.replay: - return - if _EXECUTABLE.exists() and not global_config.build_native: - return - - logger.info("Building experimental Rust recorder", executable=str(_EXECUTABLE)) - started = time.perf_counter() - process = subprocess.Popen( - _BUILD_COMMAND, - cwd=_RUST_DIR, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - assert process.stdout is not None - for raw in process.stdout: - line = raw.decode("utf-8", errors="replace").rstrip() - if line: - logger.info(line, module="rust-recorder-build") - returncode = process.wait() - if returncode != 0: - raise RuntimeError( - "Rust recorder build failed " - f"after {time.perf_counter() - started:.2f}s (exit {returncode})" - ) - if not _EXECUTABLE.exists(): - raise FileNotFoundError(f"Rust recorder build did not produce {_EXECUTABLE}") + lcm_url: str | None = None def make_plan(transports: dict[tuple[str, type], Any]) -> RustRecordingPlan: @@ -95,6 +63,7 @@ def make_plan(transports: dict[tuple[str, type], Any]) -> RustRecordingPlan: topics: dict[str, str] = {} payload_types: dict[str, type[Any]] = {} backends: set[str] = set() + lcm_urls: set[str] = set() for index, ((name, payload_type), transport) in enumerate(transports.items()): if name not in selected: @@ -108,6 +77,12 @@ def make_plan(transports: dict[tuple[str, type], Any]) -> RustRecordingPlan: continue if type(transport) is LCMTransport: backend = "lcm" + config = transport.lcm.config + if config.lcm is not None or not config.url: + raise ValueError( + "Rust recording requires an explicit LCM URL, not an external LCM connection" + ) + lcm_urls.add(config.url) elif type(transport) is ZenohTransport: backend = "zenoh" else: @@ -148,9 +123,13 @@ def make_plan(transports: dict[tuple[str, type], Any]) -> RustRecordingPlan: "narrow --record-topics or use --record-engine python." ) + if len(lcm_urls) > 1: + raise ValueError("The Rust recorder cannot record conflicting LCM URLs in one artifact") + suffix = "db" if global_config.record == "sqlite" else "mcap" return RustRecordingPlan( backend=backends.pop(), + lcm_url=next(iter(lcm_urls), None), topics=topics, streams=streams, payload_types=payload_types, @@ -181,7 +160,7 @@ def __init__(self, plan: RustRecordingPlan) -> None: self._threads: list[threading.Thread] = [] def start(self) -> None: - prepare_rust_recorder() + executable = ensure_native_package("dimos-memory-recorder") _prepare_artifact(self.plan) store = {"kind": global_config.record, "path": str(self.plan.path)} launch = { @@ -199,9 +178,10 @@ def start(self) -> None: "DIMOS_TRANSPORT": self.plan.backend, "RUST_LOG": os.environ.get("DIMOS_LOG_LEVEL", "info").lower(), } + if self.plan.lcm_url is not None: + env["LCM_DEFAULT_URL"] = self.plan.lcm_url self._process = subprocess.Popen( - [_EXECUTABLE], - cwd=_RUST_DIR, + [executable], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/dimos/experimental/memory/rust_recorder.py b/dimos/experimental/memory/rust_recorder.py index 667945cce6..8f16247dd3 100644 --- a/dimos/experimental/memory/rust_recorder.py +++ b/dimos/experimental/memory/rust_recorder.py @@ -27,8 +27,9 @@ from dimos.core.module import Module from dimos.core.native_module import NativeModule, NativeModuleConfig from dimos.core.stream import In -from dimos.memory.module import OnExisting +from dimos.experimental.memory.rust_types import RustStreamSpec from dimos.memory.store.sqlite import SqliteStore +from dimos.memory.type.recording import OnExisting from dimos.msgs.sensor_msgs.Image import Image from dimos.msgs.tf2_msgs.TFMessage import TFMessage from dimos.utils.data import backup_file @@ -39,15 +40,6 @@ _SUPPORTED_NATIVE_CODECS = {"lcm", "jpeg", "lz4+lcm"} -class RustStreamSpec(BaseModel): - """Fully resolved stream settings sent to the native process.""" - - port: str - name: str - payload_type: str - codec: str - - class RustStoreConfig(BaseModel): """Artifact path shared by native recording-store configurations.""" @@ -86,16 +78,14 @@ class RustMcapStoreConfig(RustStoreConfig): class RustRecorderConfig(NativeModuleConfig): - """Compatibility-first configuration for :class:`RustRecorder`. + """Configuration for :class:`RustRecorder`. Python owns artifact lifecycle and stream registration. The native process receives only ``store``, ``encoding_threads``, and the internally resolved ``streams`` list over stdin. """ - executable: str = "result/bin/dimos-memory-recorder" - build_command: str = "nix build -L .#dimos-memory-recorder" - cwd: str = "rust" + native_package: str | None = "dimos-memory-recorder" stdin_config: bool = True store: RustRecordingStoreConfig = Field( @@ -288,4 +278,4 @@ def _prepare_store(self, specs: list[RustStreamSpec]) -> None: def _argv(self, _topics: dict[str, str]) -> list[str]: """Launch the stdin-only recorder without topic or configuration arguments.""" - return [self.config.executable] + return [self._executable] diff --git a/dimos/experimental/memory/rust_types.py b/dimos/experimental/memory/rust_types.py new file mode 100644 index 0000000000..ee790589ea --- /dev/null +++ b/dimos/experimental/memory/rust_types.py @@ -0,0 +1,26 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Wire descriptions shared by native recording controllers.""" + +from pydantic import BaseModel + + +class RustStreamSpec(BaseModel): + """Fully resolved stream settings sent to the native process.""" + + port: str + name: str + payload_type: str + codec: str diff --git a/dimos/experimental/memory/test_rust_cli_recorder.py b/dimos/experimental/memory/test_rust_cli_recorder.py index a767baa222..5414d06574 100644 --- a/dimos/experimental/memory/test_rust_cli_recorder.py +++ b/dimos/experimental/memory/test_rust_cli_recorder.py @@ -28,7 +28,7 @@ from dimos.core.transport import LCMTransport, ZenohTransport from dimos.experimental.memory import rust_cli_recorder from dimos.experimental.memory.rust_cli_recorder import RustRecordingPlan, RustRecordingSession -from dimos.experimental.memory.rust_recorder import RustStreamSpec +from dimos.experimental.memory.rust_types import RustStreamSpec from dimos.memory.store.sqlite import SqliteStore from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.sensor_msgs.Image import Image @@ -73,16 +73,6 @@ def test_plan_uses_actual_lcm_channels_and_memory_codecs(tmp_path: Path) -> None assert plan.path == tmp_path / "memory.db" -def test_replay_does_not_build_the_native_recorder(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(global_config, "replay", True) - monkeypatch.setattr( - "dimos.experimental.memory.rust_cli_recorder.subprocess.Popen", - lambda *args, **kwargs: pytest.fail("replay must not build the recorder"), - ) - - rust_cli_recorder.prepare_rust_recorder() - - def test_plan_uses_mcap_artifact_for_zenoh(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(global_config, "record", "mcap") @@ -207,7 +197,9 @@ def _start_fake_session( monkeypatch: pytest.MonkeyPatch, process: _FakeProcess, ) -> RustRecordingSession: - monkeypatch.setattr(rust_cli_recorder, "prepare_rust_recorder", lambda: None) + monkeypatch.setattr( + rust_cli_recorder, "ensure_native_package", lambda package_id: tmp_path / "recorder" + ) monkeypatch.setattr( "dimos.experimental.memory.rust_cli_recorder.subprocess.Popen", lambda *args, **kwargs: process, @@ -217,34 +209,6 @@ def _start_fake_session( return session -def test_reuses_existing_memory_recorder_binary() -> None: - assert rust_cli_recorder._EXECUTABLE.name == "dimos-memory-recorder" - - -def test_existing_binary_skips_build(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - executable = tmp_path / "dimos-memory-recorder" - executable.touch() - monkeypatch.setattr(rust_cli_recorder, "_EXECUTABLE", executable) - monkeypatch.setattr( - "dimos.experimental.memory.rust_cli_recorder.subprocess.Popen", - lambda *args, **kwargs: pytest.fail("existing binary must not be rebuilt"), - ) - - rust_cli_recorder.prepare_rust_recorder() - - -def test_build_failure_is_reported(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - process = SimpleNamespace(stdout=BytesIO(b"build output\n"), wait=lambda: 7) - monkeypatch.setattr(rust_cli_recorder, "_EXECUTABLE", tmp_path / "missing") - monkeypatch.setattr( - "dimos.experimental.memory.rust_cli_recorder.subprocess.Popen", - lambda *args, **kwargs: process, - ) - - with pytest.raises(RuntimeError, match="Rust recorder build failed.*exit 7"): - rust_cli_recorder.prepare_rust_recorder() - - @pytest.mark.parametrize("encoding_threads", [None, 8]) def test_launch_config_uses_default_or_configured_threads( tmp_path: Path, @@ -267,7 +231,9 @@ def test_process_exit_before_ready_fails_startup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: process = _FakeProcess(stdout=b"", returncode=3) - monkeypatch.setattr(rust_cli_recorder, "prepare_rust_recorder", lambda: None) + monkeypatch.setattr( + rust_cli_recorder, "ensure_native_package", lambda package_id: tmp_path / "recorder" + ) monkeypatch.setattr( "dimos.experimental.memory.rust_cli_recorder.subprocess.Popen", lambda *args, **kwargs: process, @@ -280,7 +246,9 @@ def test_process_exit_before_ready_fails_startup( def test_readiness_timeout_fails_startup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: process = _FakeProcess(stdout=b"") monkeypatch.setattr(rust_cli_recorder, "_READY_TIMEOUT", 0.01) - monkeypatch.setattr(rust_cli_recorder, "prepare_rust_recorder", lambda: None) + monkeypatch.setattr( + rust_cli_recorder, "ensure_native_package", lambda package_id: tmp_path / "recorder" + ) monkeypatch.setattr( "dimos.experimental.memory.rust_cli_recorder.subprocess.Popen", lambda *args, **kwargs: process, @@ -317,3 +285,54 @@ def test_stop_kills_process_that_does_not_flush( assert process.signals == [signal.SIGTERM] assert process.killed + + +def test_custom_lcm_url_reaches_recorder_child(tmp_path, monkeypatch, mocker): + monkeypatch.setenv("LCM_DEFAULT_URL", "udpm://239.255.76.68:7667?ttl=0") + url = "udpm://239.255.76.69:7667?ttl=0" + transport = LCMTransport("/odom", PoseStamped, url=url) + process = _FakeProcess() + mocker.patch.object( + rust_cli_recorder, "ensure_native_package", return_value=tmp_path / "recorder" + ) + popen = mocker.patch.object(rust_cli_recorder.subprocess, "Popen", return_value=process) + session = RustRecordingSession(rust_cli_recorder.make_plan({("odom", PoseStamped): transport})) + try: + session.start() + assert popen.call_args.kwargs["env"]["LCM_DEFAULT_URL"] == url + assert "cwd" not in popen.call_args.kwargs + finally: + session.stop() + transport.stop() + + +def test_conflicting_lcm_urls_are_rejected(): + first = LCMTransport("/first", PoseStamped, url="udpm://239.255.76.68:7667?ttl=0") + second = LCMTransport("/second", PoseStamped, url="udpm://239.255.76.69:7667?ttl=0") + try: + with pytest.raises(ValueError, match="conflicting LCM URLs"): + rust_cli_recorder.make_plan( + {("first", PoseStamped): first, ("second", PoseStamped): second} + ) + finally: + first.stop() + second.stop() + + +def test_external_lcm_connection_is_rejected(): + transport = _lcm("/odom") + transport.lcm.config.lcm = transport.lcm.l + try: + with pytest.raises(ValueError, match="external LCM connection"): + rust_cli_recorder.make_plan({("odom", PoseStamped): transport}) + finally: + transport.stop() + + +def test_preparation_failure_does_not_create_artifact(tmp_path, mocker): + mocker.patch.object( + rust_cli_recorder, "ensure_native_package", side_effect=RuntimeError("build failed") + ) + with pytest.raises(RuntimeError, match="build failed"): + RustRecordingSession(_plan(tmp_path)).start() + assert not (tmp_path / "memory.db").exists() diff --git a/dimos/experimental/memory/test_rust_recorder.py b/dimos/experimental/memory/test_rust_recorder.py index c8a8e40997..11888c9e93 100644 --- a/dimos/experimental/memory/test_rust_recorder.py +++ b/dimos/experimental/memory/test_rust_recorder.py @@ -26,8 +26,8 @@ RustRecorderConfig, RustSqliteStoreConfig, ) -from dimos.memory.module import OnExisting from dimos.memory.store.sqlite import SqliteStore +from dimos.memory.type.recording import OnExisting from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.sensor_msgs.Image import Image @@ -183,14 +183,6 @@ def test_default_store_path_is_resolved_from_the_project_root() -> None: assert Path(config.store.path).name == "recording.db" -def test_native_recorder_is_built_and_run_from_the_nix_package() -> None: - config = RustRecorderConfig() - - assert config.cwd == "rust" - assert config.build_command == "nix build -L .#dimos-memory-recorder" - assert config.executable == "result/bin/dimos-memory-recorder" - - def test_invalid_codec_fails_during_preflight(tmp_path: Path, make_recorder: Any) -> None: recorder = make_recorder( SampleRustRecorder, diff --git a/dimos/experimental/memory/test_rust_recorder_e2e.py b/dimos/experimental/memory/test_rust_recorder_e2e.py index a5c4c7e025..7c7d08ff3b 100644 --- a/dimos/experimental/memory/test_rust_recorder_e2e.py +++ b/dimos/experimental/memory/test_rust_recorder_e2e.py @@ -14,7 +14,6 @@ from __future__ import annotations -import importlib.util import os from pathlib import Path import select @@ -57,7 +56,6 @@ _RUST_PACKAGE = DIMOS_PROJECT_ROOT / "dimos" / "experimental" / "memory" / "rust" _EXECUTABLE = _RUST_PACKAGE / "result" / "bin" / "dimos-memory-recorder" -_MCAP_AVAILABLE = importlib.util.find_spec("mcap") is not None class InteropRustRecorder(RustRecorder): @@ -126,10 +124,7 @@ def _free_port() -> int: "store_kind", [ "sqlite", - pytest.param( - "mcap", - marks=pytest.mark.skipif(not _MCAP_AVAILABLE, reason="mcap not installed"), - ), + "mcap", ], ) def test_rust_artifact_is_readable_by_python_memory2( @@ -149,6 +144,7 @@ def test_rust_artifact_is_readable_by_python_memory2( endpoint = f"tcp/127.0.0.1:{_free_port()}" monkeypatch.setattr(global_config, "transport", "zenoh") recorder = InteropRustRecorder( + native_package=None, executable=str(rust_recorder_executable), store=store, record_tf=False, @@ -280,8 +276,9 @@ def test_cli_recording_uses_existing_binary_for_both_formats( monkeypatch.setattr(global_config, "record_encoding_threads", 2) monkeypatch.setattr(global_config, "transport", "lcm") monkeypatch.setattr(global_config, "build_native", False) - monkeypatch.setattr(rust_cli_recorder, "_EXECUTABLE", rust_recorder_executable) - monkeypatch.setattr(rust_cli_recorder, "_RUST_DIR", _RUST_PACKAGE) + monkeypatch.setattr( + rust_cli_recorder, "ensure_native_package", lambda package_id: rust_recorder_executable + ) monkeypatch.setattr(rust_cli_recorder, "recording_dir", lambda: tmp_path) channel = f"/rust-recorder-{uuid.uuid4().hex[:8]}" publisher: LCMTransport[Imu] = LCMTransport(channel, Imu, url=lcm_url) @@ -298,11 +295,6 @@ def test_cli_recording_uses_existing_binary_for_both_formats( publisher.stop() memory: SqliteStore | McapStore - if store_kind == "mcap" and not _MCAP_AVAILABLE: - data = artifact.read_bytes() - assert data.startswith(b"\x89MCAP0\r\n") - assert data.endswith(b"\x89MCAP0\r\n") - return if store_kind == "sqlite": memory = SqliteStore(path=str(artifact)) else: @@ -322,6 +314,7 @@ def test_tf_records_over_zenoh_and_replays_through_python( endpoint = f"tcp/127.0.0.1:{_free_port()}" monkeypatch.setattr(global_config, "transport", "zenoh") recorder = RustRecorder( + native_package=None, executable=str(rust_recorder_executable), store=RustSqliteStoreConfig(path=str(artifact)), record_tf=True, diff --git a/dimos/experimental/robot/bosdyn/spot/recorder.py b/dimos/experimental/robot/bosdyn/spot/recorder.py index 8a7f0e3968..eb160dd4c2 100644 --- a/dimos/experimental/robot/bosdyn/spot/recorder.py +++ b/dimos/experimental/robot/bosdyn/spot/recorder.py @@ -29,7 +29,8 @@ from dimos.core.stream import In from dimos.experimental.robot.bosdyn.spot.config import CAMERA_STREAM_SUFFIXES -from dimos.memory.module import OnExisting, Recorder, RecorderConfig +from dimos.memory.module import Recorder, RecorderConfig +from dimos.memory.type.recording import OnExisting from dimos.msgs.nav_msgs.Odometry import Odometry from dimos.msgs.sensor_msgs.CameraInfo import CameraInfo from dimos.msgs.sensor_msgs.Image import Image diff --git a/dimos/hardware/sensors/lidar/fastlio2/module.py b/dimos/hardware/sensors/lidar/fastlio2/module.py index c179a982ac..ea384b67db 100644 --- a/dimos/hardware/sensors/lidar/fastlio2/module.py +++ b/dimos/hardware/sensors/lidar/fastlio2/module.py @@ -57,9 +57,7 @@ class FastLio2Config(NativeModuleConfig): - cwd: str | None = "cpp" - executable: str = "result/bin/fastlio2_native" - build_command: str | None = "nix build -L .#fastlio2_native" + native_package: str | None = "fastlio2" stdin_config: bool = True base_fields: frozenset[str] = frozenset({"frame_id"}) # Livox SDK hardware config. lidar_ip required; host_ip optional (auto-derived diff --git a/dimos/hardware/sensors/lidar/fastlio2/recorder.py b/dimos/hardware/sensors/lidar/fastlio2/recorder.py index f6400df354..175041bd68 100644 --- a/dimos/hardware/sensors/lidar/fastlio2/recorder.py +++ b/dimos/hardware/sensors/lidar/fastlio2/recorder.py @@ -25,7 +25,8 @@ from __future__ import annotations from dimos.core.stream import In -from dimos.memory.module import OnExisting, Recorder, RecorderConfig, pose_setter_for +from dimos.memory.module import Recorder, RecorderConfig, pose_setter_for +from dimos.memory.type.recording import OnExisting from dimos.msgs.geometry_msgs.Pose import Pose from dimos.msgs.nav_msgs.Odometry import Odometry from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 diff --git a/dimos/hardware/sensors/lidar/livox/module.py b/dimos/hardware/sensors/lidar/livox/module.py index 0af9fb7e27..46b21ff337 100644 --- a/dimos/hardware/sensors/lidar/livox/module.py +++ b/dimos/hardware/sensors/lidar/livox/module.py @@ -54,9 +54,7 @@ class Mid360Config(NativeModuleConfig): - cwd: str | None = "cpp" - executable: str = "result/bin/mid360_native" - build_command: str | None = "nix build -L .#mid360_native" + native_package: str | None = "mid360" stdin_config: bool = True base_fields: frozenset[str] = frozenset({"frame_id"}) host_ip: str | None = Field(default_factory=lambda: os.environ.get("DIMOS_MID360_HOST_IP")) diff --git a/dimos/hardware/sensors/lidar/pointlio/module.py b/dimos/hardware/sensors/lidar/pointlio/module.py index 256e6f82d9..f9381166cd 100644 --- a/dimos/hardware/sensors/lidar/pointlio/module.py +++ b/dimos/hardware/sensors/lidar/pointlio/module.py @@ -68,9 +68,7 @@ class PointLioConfig(NativeModuleConfig): - cwd: str | None = "cpp" - executable: str = "result/bin/pointlio_native" - build_command: str | None = "nix build -L .#pointlio_native" + native_package: str | None = "pointlio" stdin_config: bool = True base_fields: frozenset[str] = frozenset({"frame_id"}) # lidar_ip required; host_ip optional (auto-derived from lidar_ip's subnet). diff --git a/dimos/hardware/sensors/lidar/pointlio/recorder.py b/dimos/hardware/sensors/lidar/pointlio/recorder.py index 5ef22f8bc1..8013ec8577 100644 --- a/dimos/hardware/sensors/lidar/pointlio/recorder.py +++ b/dimos/hardware/sensors/lidar/pointlio/recorder.py @@ -25,7 +25,8 @@ from __future__ import annotations from dimos.core.stream import In -from dimos.memory.module import OnExisting, Recorder, RecorderConfig, pose_setter_for +from dimos.memory.module import Recorder, RecorderConfig, pose_setter_for +from dimos.memory.type.recording import OnExisting from dimos.msgs.geometry_msgs.Pose import Pose from dimos.msgs.nav_msgs.Odometry import Odometry from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 diff --git a/dimos/memory/module.py b/dimos/memory/module.py index 4293b561cf..e812162eec 100644 --- a/dimos/memory/module.py +++ b/dimos/memory/module.py @@ -16,7 +16,6 @@ from collections.abc import Awaitable, Callable from datetime import datetime -import enum import inspect import os from pathlib import Path @@ -39,6 +38,7 @@ from dimos.memory.stream import Stream from dimos.memory.transform import QualityWindow from dimos.memory.type.observation import EmbeddedObservation, Observation +from dimos.memory.type.recording import OnExisting from dimos.models.embedding.base import EmbeddingModel from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.sensor_msgs.Image import Image @@ -268,13 +268,6 @@ def _similarity(obs: Observation[Any]) -> float: return best.pose_stamped -class OnExisting(str, enum.Enum): - OVERWRITE = "overwrite" - ERROR = "error" - BACKUP = "backup" - APPEND = "append" - - class RecorderConfig(MemoryModuleConfig): on_existing: OnExisting = OnExisting.BACKUP backup_keep_last: int = Field(default=10, ge=0) diff --git a/dimos/memory/store/mcap.py b/dimos/memory/store/mcap.py index 4cb75ec0a4..381bfe13bc 100644 --- a/dimos/memory/store/mcap.py +++ b/dimos/memory/store/mcap.py @@ -14,8 +14,8 @@ """Read-only memory store backed by an mcap file. -Generic and robot-independent. JPEG channels decode automatically because their -payload type is fixed. Other formats use a caller-supplied ``codecs`` map (wire +Generic and robot-independent. JPEG channels and native DimOS message channels +decode automatically. Other formats use a caller-supplied ``codecs`` map (wire topic -> codec), while ``streams`` may map friendly stream names to topics. See ``dimos.robot.unitree.go2.dds.store.Go2McapStore`` for the Go2 DDS wiring. @@ -28,16 +28,22 @@ from collections.abc import Iterator, Mapping from dataclasses import replace from functools import partial +import re from typing import Any, Protocol, runtime_checkable +from mcap.reader import make_reader + from dimos.memory.backend import Backend from dimos.memory.codecs.base import codec_for from dimos.memory.codecs.jpeg import JpegCodec +from dimos.memory.codecs.lcm import LcmCodec +from dimos.memory.codecs.lz4 import Lz4Codec from dimos.memory.notifier.subject import SubjectNotifier from dimos.memory.observationstore.base import ObservationStore, ObservationStoreConfig from dimos.memory.store.base import Store, StoreConfig from dimos.memory.type.filter import StreamQuery from dimos.memory.type.observation import Observation +from dimos.msgs.helpers import resolve_msg_type @runtime_checkable @@ -104,8 +110,6 @@ def name(self) -> str: return self.config.name def _iter(self, reverse: bool = False) -> Iterator[Observation[Any]]: - from mcap.reader import make_reader # optional mcap dependency - decode, dtype, n = self._codec.decode, self._codec.payload_type, self._count with open(self._path, "rb") as f: msgs = make_reader(f).iter_messages(topics=[self._topic], reverse=reverse) @@ -173,8 +177,6 @@ def __init__( streams: dict[str, str] | None = None, **kwargs: Any, ) -> None: - from mcap.reader import make_reader # optional mcap dependency - super().__init__(**kwargs) self._codecs = dict(codecs or {}) name_of = {topic: name for name, topic in (streams or {}).items()} # topic -> override @@ -193,6 +195,19 @@ def __init__( name = name_of.get(ch.topic) or _slug(ch.topic) if ch.topic not in self._codecs and ch.message_encoding == "jpeg": self._codecs[ch.topic] = JpegCodec() + if ch.topic not in self._codecs and ch.message_encoding in {"lcm", "lz4+lcm"}: + # Only built-in message names may select a decoder. Never + # import arbitrary Python modules named by artifact metadata. + match = re.fullmatch( + r"dimos\.msgs\.([a-z][a-z0-9_]*_msgs)\.([A-Z][A-Za-z0-9]*)\.\2", + ch.metadata.get("dimos.payload_type", ""), + ) + payload_type = resolve_msg_type(f"{match[1]}.{match[2]}") if match else None + if payload_type is not None: + codec = LcmCodec(payload_type) + self._codecs[ch.topic] = ( + Lz4Codec(codec) if ch.message_encoding == "lz4+lcm" else codec + ) self._stream_topic[name] = ch.topic self._available[name] = count self._observation_uses_publish_time[name] = ( diff --git a/dimos/memory/store/test_mcap.py b/dimos/memory/store/test_mcap.py index e6e3496e61..77e917a680 100644 --- a/dimos/memory/store/test_mcap.py +++ b/dimos/memory/store/test_mcap.py @@ -31,7 +31,8 @@ mcap_writer = pytest.importorskip("mcap.writer", reason="mcap not installed") -def test_lcm_channel_decodes_with_explicit_codec(tmp_path: Path) -> None: +@pytest.mark.parametrize("explicit", [False, True]) +def test_native_lcm_channel_decodes(tmp_path: Path, explicit: bool) -> None: path = tmp_path / "recording.mcap" expected = Imu( ts=12.5, @@ -64,7 +65,7 @@ def test_lcm_channel_decodes_with_explicit_codec(tmp_path: Path) -> None: ) writer.finish() - with McapStore(path=str(path), codecs={"imu": LcmCodec(Imu)}) as store: + with McapStore(path=str(path), codecs={"imu": LcmCodec(Imu)} if explicit else None) as store: assert store.list_streams() == ["imu"] observation: Observation[Imu] = store.stream("imu").order_by("ts").first() assert observation.ts == 11.5 @@ -75,7 +76,8 @@ def test_lcm_channel_decodes_with_explicit_codec(tmp_path: Path) -> None: assert latest_observation.data.lcm_encode() == expected.lcm_encode() -def test_wrapped_codec_decodes_with_explicit_codec(tmp_path: Path) -> None: +@pytest.mark.parametrize("explicit", [False, True]) +def test_native_wrapped_codec_decodes(tmp_path: Path, explicit: bool) -> None: path = tmp_path / "recording.mcap" expected = Imu( ts=12.5, @@ -103,7 +105,7 @@ def test_wrapped_codec_decodes_with_explicit_codec(tmp_path: Path) -> None: ) writer.finish() - with McapStore(path=str(path), codecs={"imu": codec}) as store: + with McapStore(path=str(path), codecs={"imu": codec} if explicit else None) as store: observation: Observation[Imu] = store.stream("imu").first() assert observation.ts == 12.5 assert observation.data.lcm_encode() == expected.lcm_encode() diff --git a/dimos/memory/type/recording.py b/dimos/memory/type/recording.py new file mode 100644 index 0000000000..489325030c --- /dev/null +++ b/dimos/memory/type/recording.py @@ -0,0 +1,24 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Policy for recording to an existing artifact.""" + +import enum + + +class OnExisting(str, enum.Enum): + OVERWRITE = "overwrite" + ERROR = "error" + BACKUP = "backup" + APPEND = "append" diff --git a/dimos/native_packages.json b/dimos/native_packages.json new file mode 100644 index 0000000000..308c5aa119 --- /dev/null +++ b/dimos/native_packages.json @@ -0,0 +1,22 @@ +{ + "dimos-memory-recorder": { + "flake_dir": "dimos/experimental/memory/rust", + "attribute": "dimos-memory-recorder", + "executable": "dimos-memory-recorder" + }, + "mid360": { + "flake_dir": "dimos/hardware/sensors/lidar/livox/cpp", + "attribute": "mid360_native", + "executable": "mid360_native" + }, + "pointlio": { + "flake_dir": "dimos/hardware/sensors/lidar/pointlio/cpp", + "attribute": "pointlio_native", + "executable": "pointlio_native" + }, + "fastlio2": { + "flake_dir": "dimos/hardware/sensors/lidar/fastlio2/cpp", + "attribute": "fastlio2_native", + "executable": "fastlio2_native" + } +} diff --git a/docs/adr/0001-native-package-preparation.md b/docs/adr/0001-native-package-preparation.md new file mode 100644 index 0000000000..6b1403ed64 --- /dev/null +++ b/docs/adr/0001-native-package-preparation.md @@ -0,0 +1,15 @@ +# Share native package preparation through Nix + +DimOS native launchers depended on build directories beside Python modules, +which are absent from pip installations. The recorder, Livox Mid360, PointLIO, +and FastLIO2 now share a package catalog and automatic preparation: Nix reuses +local outputs, substitutes from Cachix, or builds the selected sources. Wheels +and source distributions carry an immutable source revision; editable +checkouts use local inputs. + +We retain Nix as a runtime prerequisite instead of distributing portable +binaries in Python wheels. This reuses the existing CI publisher and preserves +native dependency closures, at the cost of Nix installation and potentially +slow first-use compilation. Builds are automatic, with conservative +parallelism; process lifecycles remain owned by NativeModule or the recording +context. GPU-specific and Cargo-only packaging remains separate work. diff --git a/docs/usage/native_modules.md b/docs/usage/native_modules.md index 8772e71b32..c05e66972b 100644 --- a/docs/usage/native_modules.md +++ b/docs/usage/native_modules.md @@ -79,7 +79,8 @@ When `stop()` is called, the process receives SIGTERM. If it doesn't exit within | Field | Type | Default | Description | |--------------------|------------------|---------------|-------------------------------------------------------------| -| `executable` | `str` | *(required)* | Path to the native binary (relative to `cwd` if set) | +| `native_package` | `str \| None` | `None` | Package ID from the native catalog; automatically prepared by Nix | +| `executable` | `str` | `""` | Explicit binary path; required when `native_package` is unset | | `build_command` | `str \| None` | `None` | Shell command to run if executable is missing (auto-build) | | `cwd` | `str \| None` | `None` | Working directory for build and runtime. Relative paths are resolved against the Python file defining the module | | `extra_args` | `list[str]` | `[]` | Additional CLI arguments appended after auto-generated ones | @@ -267,32 +268,70 @@ autoconnect( ) ``` -## Auto Building - -If `build_command` is set in the module config, and the executable doesn't exist when `start()` is called, NativeModule runs the build command automatically. -Build output is streamed line by line through structlog at `info`, with stderr merged into -stdout. `nix build` prints no build logs unless `-L` is passed, so the built-in modules all -include it. - -```python skip -class MyLidarConfig(NativeModuleConfig): - cwd: str | None = "cpp" - executable: str = "result/bin/my_lidar" - build_command: str | None = "nix build -L .#my_lidar" +## Preparing native packages + +The Rust recorder, Livox Mid360, PointLIO, and FastLIO2 use shared native +package preparation. First use evaluates the package's Nix inputs, reuses a +local result, downloads a matching result from the configured binary cache, +or builds from source. Compilation uses one Nix build job with two cores. +Preparation failures stop startup before the affected process launches. + +| Term | Meaning | +|------|---------| +| Native package | A versioned executable and its runtime dependencies | +| Prepared executable | A native package's installed, runnable binary | +| Recording engine | The implementation that records streams, currently Python or Rust | + +Nix with flakes enabled is required. A pip-installed DimOS uses the immutable +source revision embedded in its wheel or source distribution. An editable +checkout uses its local native sources. Nix determines reuse from the build +inputs: unrelated Python edits do not force recompilation. Add newly created +native source files to Git so local Git flakes include them. + +To prepare before starting hardware: + +```bash skip +dimos native prepare dimos-memory-recorder +dimos native prepare mid360 +dimos native prepare pointlio +dimos native prepare fastlio2 ``` -`cwd` is used for both the build command and the runtime subprocess. Relative paths are resolved against the directory of the Python file that defines the module +The same commands prepare local edits in a checkout. `--build-native` reruns +preparation during module building; it does not bypass Nix's reuse of identical +outputs. Executables and their dependencies stay in the Nix store, with output +links under the DimOS cache root. Runtime processes do not need a source-tree +working directory. Different native builds have independent output links. -If the executable already exists, the build step is skipped entirely. +Package declarations live in `dimos/native_packages.json` and are consumed by +both runtime preparation and the CI publisher. Set `native_package` on a +`NativeModuleConfig` to select one; do not also set `executable`, `cwd`, or +`build_command`. For an explicitly provisioned binary, set `native_package=None` +and supply `executable`. -### Faster builds via the Cachix substituter +RealSense, Cargo-only modules, and GPU-specific DimSLAM retain their existing +build workflows. Modules using `build_command` build when their executable is +missing, when `auto_build` is enabled, or when `--build-native` is requested. +Their relative paths resolve against the Python module defining them. -Nix-built native modules can be substituted from the `dimensionalos` Cachix -cache (the same substituter CI uses) instead of compiled from source. Opt in -locally to skip cold compiles when the cache has them: +### Configure Cachix -``` -# ~/.config/nix/nix.conf (single-user) or /etc/nix/nix.conf (multi-user) +DimOS requests the same cache CI publishes to on each Nix invocation. For a +multi-user installation whose daemon does not trust user-supplied caches, an +administrator must configure the cache in `/etc/nix/nix.conf`: + +```text extra-substituters = https://dimensionalos.cachix.org extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= ``` + +Follow Nix's installation instructions when configuring daemon trust. DimOS +does not edit system Nix configuration. If Nix warns that it ignored an +untrusted substituter, configure daemon trust to avoid compiling packages +that are already available from Cachix. The shared +publishing job currently covers x86-64 Linux; other supported targets may +compile locally. An absent cached result permits a source build; download or +build failures are reported rather than retried through a separate installer. + +See [Nix installation](/docs/installation/nix.md) and the +[preparation decision](/docs/adr/0001-native-package-preparation.md). diff --git a/docs/usage/recording.md b/docs/usage/recording.md index a8e6daea0e..ddf20a4e4a 100644 --- a/docs/usage/recording.md +++ b/docs/usage/recording.md @@ -35,8 +35,20 @@ other specialized transports before creating an artifact. Narrow `--record-topics` or use the Python engine when a selection contains one of those transports. Payloads must also be dimOS LCM message types. -The native process must report ready within 10 seconds, so build, configuration, -and subscription failures stop startup. If it exits unexpectedly after startup, +The engine automatically prepares `dimos-memory-recorder` through the shared +[Nix package workflow](/docs/usage/native_modules.md#preparing-native-packages). +This works from pip installations and editable checkouts; a cache miss builds +from source. To prepare ahead of time, run `dimos native prepare dimos-memory-recorder`. +MCAP reading is included in the core Python dependencies. Built-in DimOS +message channels decode from their recorded type and codec metadata, so +`dimos mem rerun memory.mcap` can render them directly. + +Selected LCM streams must use one explicit connection URL. The recorder uses +that URL even when it differs from `LCM_DEFAULT_URL`. Conflicting URLs and +externally supplied LCM connections are rejected. + +After preparation, the native process must report ready within 10 seconds. +Preparation, configuration, and subscription failures stop startup. If it exits unexpectedly after startup, the error is logged and the rest of `dimos run` continues. Normal shutdown sends SIGTERM and lets the existing native module runtime flush the artifact. There is no automatic fallback to Python. diff --git a/pyproject.toml b/pyproject.toml index 5b16d2977c..813bbe8559 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,7 @@ classifiers = [ ] dependencies = [ + "mcap>=1.2.0", # Core memory CLI reads native MCAP recordings. # Transport Protocols "dimos-lcm>=0.1.3", "eclipse-zenoh>=1.9.0,<2.0", @@ -276,7 +277,6 @@ unitree-dds = [ "dimos[unitree]", "unitree-sdk2py-dimos>=1.0.2", "cyclonedds>=0.10.5", - "mcap>=1.2.0", # decode Go2 DDS mcap recordings (go2 dds store) ] manipulation = [ @@ -502,10 +502,6 @@ lint = [ tests-self-hosted = [ {include-group = "tests"}, "dimos[agents,perception,manipulation,sim,unitree,misc]", - # go2 dds store tests decode mcap (pure-Python). The full `unitree-dds` extra - # pulls `cyclonedds`, whose wheel needs a CycloneDDS C lib the ros-dev image - # doesn't expose to the build — and these tests never open a live DDS link. - "mcap>=1.2.0", # Needed to compile the in-tree extensions. "pybind11>=2.12", "dim-odom>=0.4.1", diff --git a/setup.py b/setup.py index 6eca175e1e..375c35fd29 100644 --- a/setup.py +++ b/setup.py @@ -13,14 +13,47 @@ # limitations under the License. import fnmatch +import json import os from pathlib import Path +import re import struct +import subprocess import sys from pybind11.setup_helpers import Pybind11Extension, build_ext from setuptools import find_packages, setup from setuptools.command.build_py import build_py as _build_py +from setuptools.command.sdist import sdist as _sdist + + +def native_revision() -> str: + """Preserve provenance through sdist -> wheel, including builds without Git.""" + root = Path(__file__).resolve().parent + metadata = root / "dimos" / "_native_revision.json" + if metadata.is_file(): + revision = json.loads(metadata.read_text())["revision"] + elif os.environ.get("DIMOS_BUILD_REVISION"): + revision = os.environ["DIMOS_BUILD_REVISION"] + else: + revision = subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], text=True + ).strip() + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise RuntimeError("A full Git source revision is required to package native executables") + return revision + + +def write_native_revision(directory: str | Path) -> None: + path = Path(directory) / "dimos" / "_native_revision.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"revision": native_revision()}) + "\n") + + +class sdist(_sdist): + def make_release_tree(self, base_dir: str, files: list[str]) -> None: + super().make_release_tree(base_dir, files) + write_native_revision(base_dir) def python_is_macos_universal_binary(executable: str | None = None) -> bool: @@ -93,6 +126,7 @@ def find_package_modules(self, package, package_dir): def run(self): super().run() if not getattr(self, "editable_mode", False): + write_native_revision(self.build_lib) self._copy_relay_dist() def _copy_relay_dist(self): @@ -166,5 +200,5 @@ def _copy_relay_dist(self): packages=find_packages(), package_dir={"": "."}, ext_modules=ext_modules, - cmdclass={"build_ext": build_ext, "build_py": build_py}, + cmdclass={"build_ext": build_ext, "build_py": build_py, "sdist": sdist}, ) diff --git a/uv.lock b/uv.lock index b27e189c22..5001b678fe 100644 --- a/uv.lock +++ b/uv.lock @@ -1771,6 +1771,7 @@ dependencies = [ { name = "lazy-loader" }, { name = "llvmlite" }, { name = "lz4" }, + { name = "mcap" }, { name = "numba" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -2066,7 +2067,6 @@ unitree-dds = [ { name = "langchain-ollama" }, { name = "langchain-openai" }, { name = "lap" }, - { name = "mcap" }, { name = "moondream" }, { name = "ollama" }, { name = "omegaconf" }, @@ -2263,7 +2263,6 @@ tests-self-hosted = [ { name = "langchain-openai" }, { name = "lap" }, { name = "maturin" }, - { name = "mcap" }, { name = "md-babel-py" }, { name = "moondream" }, { name = "mujoco" }, @@ -2364,7 +2363,7 @@ requires-dist = [ { name = "manifold3d", marker = "extra == 'apriltag'", specifier = ">=2.5.0" }, { name = "matplotlib", marker = "extra == 'graspgenx'", specifier = ">=3.7.1" }, { name = "matplotlib", marker = "extra == 'manipulation'", specifier = ">=3.7.1" }, - { name = "mcap", marker = "extra == 'unitree-dds'", specifier = ">=1.2.0" }, + { name = "mcap", specifier = ">=1.2.0" }, { name = "moondream", marker = "extra == 'perception'" }, { name = "mujoco", marker = "extra == 'sim'", specifier = ">=3.3.4" }, { name = "numba", specifier = ">=0.60.0" }, @@ -2601,7 +2600,6 @@ tests-self-hosted = [ { name = "langchain-openai", specifier = ">=1,<2" }, { name = "lap", specifier = ">=0.5.12" }, { name = "maturin", specifier = ">=1.7" }, - { name = "mcap", specifier = ">=1.2.0" }, { name = "md-babel-py", specifier = ">=1.4.0" }, { name = "moondream" }, { name = "mujoco", specifier = ">=3.3.4" },