From 5463811a86b84d117ce127fa43ee42fce1df159c Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 9 Sep 2026 22:27:46 -0700 Subject: [PATCH 1/5] fix(native): build bundled SDK sources in writable caches --- dimos/core/native_module.py | 17 +++- dimos/core/native_sources.py | 83 +++++++++++++++++++ dimos/core/test_native_sources.py | 44 ++++++++++ dimos/experimental/memory/rust_recorder.py | 1 + .../sensors/camera/realsense/camera.py | 1 + .../hardware/sensors/lidar/fastlio2/module.py | 1 + dimos/hardware/sensors/lidar/livox/module.py | 1 + .../hardware/sensors/lidar/pointlio/module.py | 1 + .../sensors/lidar/virtual_mid360/module.py | 4 +- dimos/mapping/dim_slam/dim_slam.py | 1 + dimos/mapping/dim_slam/rust/flake.nix | 10 +-- dimos/mapping/ray_tracing/module.py | 4 +- .../nav_3d/mls_planner/mls_planner_native.py | 4 +- dimos_build.py | 46 ++++++++++ setup.py | 12 ++- 15 files changed, 214 insertions(+), 16 deletions(-) create mode 100644 dimos/core/native_sources.py create mode 100644 dimos/core/test_native_sources.py diff --git a/dimos/core/native_module.py b/dimos/core/native_module.py index 3f532419fb..0e3694d88c 100644 --- a/dimos/core/native_module.py +++ b/dimos/core/native_module.py @@ -54,12 +54,14 @@ class MyCppModule(NativeModule): import time from typing import IO, Any +from filelock import FileLock from pydantic import Field, model_validator -from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT +from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT, DIMOS_PROJECT_ROOT from dimos.core.core import rpc from dimos.core.global_config import global_config from dimos.core.module import Module, ModuleConfig +from dimos.core.native_sources import native_source_root from dimos.core.transport_factory import session_config from dimos.protocol.service.spec import SessionConfig from dimos.utils.logging_config import setup_logger @@ -120,6 +122,7 @@ class NativeModuleConfig(ModuleConfig): executable: str build_command: str | None = None cwd: str | None = None + bundled_sources: bool = False extra_args: list[str] = Field(default_factory=list) extra_env: dict[str, str] = Field(default_factory=dict) # Session settings for this module alone, e.g. opening it as the zenoh router @@ -223,7 +226,13 @@ def __init__(self, **kwargs: Any) -> None: if self.config.cwd is not None and not Path(self.config.cwd).is_absolute(): base_dir = Path(inspect.getfile(type(self))).resolve().parent + if self.config.bundled_sources: + base_dir = native_source_root() / base_dir.relative_to(DIMOS_PROJECT_ROOT) self.config.cwd = str(base_dir / self.config.cwd) + if self.config.bundled_sources: + self.config.extra_env.setdefault( + "CARGO_TARGET_DIR", str(Path(self.config.cwd) / "target") + ) if not Path(self.config.executable).is_absolute(): # The spawn runs from the executable's own directory, so a relative # path has to be resolved before then or it resolves against itself. @@ -233,7 +242,11 @@ def __init__(self, **kwargs: Any) -> None: @rpc def build(self) -> None: super().build() - self._maybe_build() + if self.config.bundled_sources and self.config.cwd: + with FileLock(str(Path(self.config.cwd) / ".build.lock")): + self._maybe_build() + else: + self._maybe_build() def _spawn_env(self) -> dict[str, str]: env = {**os.environ, **self.config.extra_env} diff --git a/dimos/core/native_sources.py b/dimos/core/native_sources.py new file mode 100644 index 0000000000..50de4ef94b --- /dev/null +++ b/dimos/core/native_sources.py @@ -0,0 +1,83 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Writable native build sources belonging to the installed SDK.""" + +from hashlib import sha256 +import os +from pathlib import Path +import platform +import shutil +import subprocess +import tarfile +import tempfile + +from filelock import FileLock + +from dimos.constants import CACHE_DIR, DIMOS_PROJECT_ROOT + + +def unpack_sources(bundle: Path, cache: Path) -> Path: + digest = sha256(bundle.read_bytes()).hexdigest() + root = cache / f"{digest}-{platform.system()}-{platform.machine()}" + cache.mkdir(parents=True, exist_ok=True) + with FileLock(str(root) + ".lock"): + if (root / ".git").is_dir(): + return root + temporary = Path(tempfile.mkdtemp(prefix="native-", dir=cache)) + try: + with tarfile.open(bundle) as archive: + archive.extractall(temporary, filter="data") + # Existing nested flakes use repository-relative inputs. A local + # snapshot lets Nix include their sibling sources without fetching + # another DimOS checkout or writing into site-packages. + (temporary / ".gitignore").write_text("target/\nresult\nresult-*\n.build.lock\n") + git_env = { + key: value for key, value in os.environ.items() if not key.startswith("GIT_") + } + git_env.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") + for args in ( + ["init", "--quiet"], + ["add", "."], + [ + "-c", + "user.name=DimOS", + "-c", + "user.email=build@dimos.invalid", + "-c", + "core.hooksPath=/dev/null", + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "Native SDK sources", + ], + ): + subprocess.run( + ["git", *args], cwd=temporary, env=git_env, check=True, capture_output=True + ) + temporary.rename(root) + finally: + if temporary.exists(): + shutil.rmtree(temporary) + return root + + +def native_source_root() -> Path: + bundle = Path(__file__).resolve().parents[1] / "_native_sources.tar" + if bundle.is_file(): + return unpack_sources(bundle, CACHE_DIR / "native") + if (DIMOS_PROJECT_ROOT / "Cargo.toml").is_file(): + return DIMOS_PROJECT_ROOT + raise FileNotFoundError("The DimOS SDK is missing its native sources. Reinstall the SDK.") diff --git a/dimos/core/test_native_sources.py b/dimos/core/test_native_sources.py new file mode 100644 index 0000000000..d665a002a4 --- /dev/null +++ b/dimos/core/test_native_sources.py @@ -0,0 +1,44 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tarfile + +from dimos.core.native_sources import unpack_sources + + +def test_native_sources_are_reusable_writable_snapshots(tmp_path): + source = tmp_path / "Cargo.toml" + source.write_text("[workspace]\nmembers = []\n") + bundle = tmp_path / "sources.tar" + with tarfile.open(bundle, "w") as archive: + archive.add(source, arcname="Cargo.toml") + root = unpack_sources(bundle, tmp_path / "cache") + assert (root / "Cargo.toml").read_text() == source.read_text() + assert (root / ".git").is_dir() + (root / "target").mkdir() + assert unpack_sources(bundle, tmp_path / "cache") == root + assert (root / "target").is_dir() + + +def test_source_content_selects_separate_build_roots(tmp_path): + source = tmp_path / "Cargo.toml" + bundle = tmp_path / "sources.tar" + roots = [] + for version in ("one", "two"): + source.write_text(version) + with tarfile.open(bundle, "w") as archive: + archive.add(source, arcname="Cargo.toml") + roots.append(unpack_sources(bundle, tmp_path / "cache")) + assert roots[0] != roots[1] + assert (roots[0] / "Cargo.toml").read_text() == "one" diff --git a/dimos/experimental/memory/rust_recorder.py b/dimos/experimental/memory/rust_recorder.py index 667945cce6..029718fdc5 100644 --- a/dimos/experimental/memory/rust_recorder.py +++ b/dimos/experimental/memory/rust_recorder.py @@ -95,6 +95,7 @@ class RustRecorderConfig(NativeModuleConfig): executable: str = "result/bin/dimos-memory-recorder" build_command: str = "nix build -L .#dimos-memory-recorder" + bundled_sources: bool = True cwd: str = "rust" stdin_config: bool = True diff --git a/dimos/hardware/sensors/camera/realsense/camera.py b/dimos/hardware/sensors/camera/realsense/camera.py index 279a64f016..59155cc8b0 100644 --- a/dimos/hardware/sensors/camera/realsense/camera.py +++ b/dimos/hardware/sensors/camera/realsense/camera.py @@ -34,6 +34,7 @@ class RealSenseCameraConfig(NativeModuleConfig, DepthCameraConfig): + bundled_sources: bool = True cwd: str | None = "rust" executable: str = "target/release/realsense_native" # Own flake: librealsense2 isn't in the root shell. diff --git a/dimos/hardware/sensors/lidar/fastlio2/module.py b/dimos/hardware/sensors/lidar/fastlio2/module.py index c179a982ac..0d084cabb7 100644 --- a/dimos/hardware/sensors/lidar/fastlio2/module.py +++ b/dimos/hardware/sensors/lidar/fastlio2/module.py @@ -57,6 +57,7 @@ class FastLio2Config(NativeModuleConfig): + bundled_sources: bool = True cwd: str | None = "cpp" executable: str = "result/bin/fastlio2_native" build_command: str | None = "nix build -L .#fastlio2_native" diff --git a/dimos/hardware/sensors/lidar/livox/module.py b/dimos/hardware/sensors/lidar/livox/module.py index 0af9fb7e27..b746991bcf 100644 --- a/dimos/hardware/sensors/lidar/livox/module.py +++ b/dimos/hardware/sensors/lidar/livox/module.py @@ -54,6 +54,7 @@ class Mid360Config(NativeModuleConfig): + bundled_sources: bool = True cwd: str | None = "cpp" executable: str = "result/bin/mid360_native" build_command: str | None = "nix build -L .#mid360_native" diff --git a/dimos/hardware/sensors/lidar/pointlio/module.py b/dimos/hardware/sensors/lidar/pointlio/module.py index 256e6f82d9..6850924028 100644 --- a/dimos/hardware/sensors/lidar/pointlio/module.py +++ b/dimos/hardware/sensors/lidar/pointlio/module.py @@ -68,6 +68,7 @@ class PointLioConfig(NativeModuleConfig): + bundled_sources: bool = True cwd: str | None = "cpp" executable: str = "result/bin/pointlio_native" build_command: str | None = "nix build -L .#pointlio_native" diff --git a/dimos/hardware/sensors/lidar/virtual_mid360/module.py b/dimos/hardware/sensors/lidar/virtual_mid360/module.py index 51370e7a00..cc0eab658e 100644 --- a/dimos/hardware/sensors/lidar/virtual_mid360/module.py +++ b/dimos/hardware/sensors/lidar/virtual_mid360/module.py @@ -36,7 +36,6 @@ from pydantic import Field -from dimos.constants import DIMOS_PROJECT_ROOT from dimos.core.core import rpc from dimos.core.native_module import NativeModule, NativeModuleConfig from dimos.utils.logging_config import setup_logger @@ -55,9 +54,10 @@ class VirtualMid360Config(NativeModuleConfig): + bundled_sources: bool = True cwd: str | None = "." # The crate is a workspace member, so cargo builds into the repo-root target dir. - executable: str = str(DIMOS_PROJECT_ROOT / "target" / "release" / "virtual_mid360") + executable: str = "target/release/virtual_mid360" build_command: str | None = "cargo build --release" # The rust binary reads its config as a JSON object on stdin (required). stdin_config: bool = True diff --git a/dimos/mapping/dim_slam/dim_slam.py b/dimos/mapping/dim_slam/dim_slam.py index f952cb7823..fc66d82975 100644 --- a/dimos/mapping/dim_slam/dim_slam.py +++ b/dimos/mapping/dim_slam/dim_slam.py @@ -120,6 +120,7 @@ class SourceConfig(BaseModel): class DimSlamConfig(NativeModuleConfig): + bundled_sources: bool = True cwd: str | None = "rust" executable: str = "result/bin/dim_slam" build_command: str | None = Field( diff --git a/dimos/mapping/dim_slam/rust/flake.nix b/dimos/mapping/dim_slam/rust/flake.nix index ad59208687..3058d8c5ec 100644 --- a/dimos/mapping/dim_slam/rust/flake.nix +++ b/dimos/mapping/dim_slam/rust/flake.nix @@ -7,15 +7,11 @@ cu-vslam-rs.url = "github:jeff-hykin/cu_vslam_rs"; cu-vslam-rs.inputs.nixpkgs.follows = "nixpkgs"; cu-vslam-rs.inputs.flake-utils.follows = "flake-utils"; - # Relative git+file: will be deprecated (nix#12281) but there's no - # viable alternative for reaching local path deps outside the flake dir currently - # presumably an alternative will be added before this is removed. - dimos-repo = { url = "git+file:../../../.."; flake = false; }; crate2nix.url = "github:nix-community/crate2nix"; crate2nix.inputs.nixpkgs.follows = "nixpkgs"; }; - outputs = { self, nixpkgs, flake-utils, cu-vslam-rs, dimos-repo, crate2nix }: + outputs = { self, nixpkgs, flake-utils, cu-vslam-rs, crate2nix }: # Not eachDefaultSystem: nixpkgs 26.11 dropped x86_64-darwin, and merely naming # it is an eval error. flake-utils.lib.eachSystem [ "aarch64-darwin" "aarch64-linux" "x86_64-linux" ] (system: @@ -42,8 +38,8 @@ cp ${./build.rs} $out/dimos/mapping/dim_slam/rust/build.rs mkdir -p $out/native/rust - cp -r ${dimos-repo}/native/rust/dimos-module $out/native/rust/dimos-module - cp -r ${dimos-repo}/native/rust/dimos-module-macros $out/native/rust/dimos-module-macros + cp -r ${../../../../native/rust/dimos-module} $out/native/rust/dimos-module + cp -r ${../../../../native/rust/dimos-module-macros} $out/native/rust/dimos-module-macros ''; # One derivation per crate rather than one vendored blob, so a dependency bump diff --git a/dimos/mapping/ray_tracing/module.py b/dimos/mapping/ray_tracing/module.py index d18e042f84..a57524e956 100644 --- a/dimos/mapping/ray_tracing/module.py +++ b/dimos/mapping/ray_tracing/module.py @@ -17,7 +17,6 @@ from typing import TYPE_CHECKING -from dimos.constants import DIMOS_PROJECT_ROOT from dimos.core.native_module import NativeModule, NativeModuleConfig from dimos.core.stream import In, Out from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped @@ -32,9 +31,10 @@ class RayTracingVoxelMapConfig(NativeModuleConfig): + bundled_sources: bool = True cwd: str | None = "rust" # The crate is a workspace member, so cargo builds into the repo-root target dir. - executable: str = str(DIMOS_PROJECT_ROOT / "target" / "release" / "voxel_ray_tracing") + executable: str = "target/release/voxel_ray_tracing" build_command: str | None = "cargo build --release" stdin_config: bool = True diff --git a/dimos/navigation/nav_3d/mls_planner/mls_planner_native.py b/dimos/navigation/nav_3d/mls_planner/mls_planner_native.py index 595ece115c..c6abd948a3 100644 --- a/dimos/navigation/nav_3d/mls_planner/mls_planner_native.py +++ b/dimos/navigation/nav_3d/mls_planner/mls_planner_native.py @@ -16,7 +16,6 @@ from __future__ import annotations -from dimos.constants import DIMOS_PROJECT_ROOT from dimos.core.native_module import NativeModule, NativeModuleConfig from dimos.core.stream import In, Out from dimos.msgs.geometry_msgs.PointStamped import PointStamped @@ -28,9 +27,10 @@ class MLSPlannerNativeConfig(NativeModuleConfig): + bundled_sources: bool = True cwd: str | None = "rust" # The crate is a workspace member, so cargo builds into the repo-root target dir. - executable: str = str(DIMOS_PROJECT_ROOT / "target" / "release" / "mls_planner") + executable: str = "target/release/mls_planner" build_command: str | None = "cargo build --release" stdin_config: bool = True diff --git a/dimos_build.py b/dimos_build.py index 8d2e30cfb1..974b5a631b 100644 --- a/dimos_build.py +++ b/dimos_build.py @@ -14,9 +14,16 @@ """Build-time helpers that do not import the DimOS runtime.""" +import os from pathlib import Path import shutil import subprocess +import tarfile + +try: + import tomllib +except ImportError: + import tomli as tomllib def ensure_web_dist(root: Path) -> None: @@ -34,3 +41,42 @@ def ensure_web_dist(root: Path) -> None: subprocess.run([deno, "task", "--cwd", str(source), "build"], check=True) if not (source / "dist" / artifact).is_file(): raise RuntimeError(f"Deno build did not produce {source / 'dist' / artifact}") + + +def native_files(root: Path) -> list[Path]: + """Collect native build roots, preserving workspace and sibling dependencies.""" + workspace = tomllib.loads((root / "Cargo.toml").read_text())["workspace"] + roots = {root / member for member in workspace["members"]} + roots.update({root / "native", root / "dimos/hardware/sensors/lidar/common"}) + for directory, dirs, files in os.walk(root / "dimos"): + dirs[:] = [ + name + for name in dirs + if name not in {"target", "build", "node_modules", ".git", "__pycache__"} + ] + if "flake.nix" in files or "Cargo.toml" in files: + roots.add(Path(directory)) + paths = {root / "Cargo.toml", root / "Cargo.lock"} + for source in roots: + for directory, dirs, files in os.walk(source): + dirs[:] = [ + name + for name in dirs + if name not in {"target", "build", "node_modules", ".git", "result", "__pycache__"} + ] + for name in files: + path = Path(directory) / name + if not path.is_symlink() and path.suffix not in {".pyc", ".pcap", ".db", ".rrd"}: + paths.add(path) + return sorted(path for path in paths if path.is_file()) + + +def bundle_native_sources(root: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(destination, "w") as archive: + for path in native_files(root): + info = archive.gettarinfo(str(path), arcname=str(path.relative_to(root))) + info.uid = info.gid = info.mtime = 0 + info.uname = info.gname = "" + with path.open("rb") as source: + archive.addfile(info, source) diff --git a/setup.py b/setup.py index e2f0c441df..bbc779fd97 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ # PEP 517 executes this file without adding the source root to sys.path. sys.path.insert(0, str(Path(__file__).parent)) -from dimos_build import ensure_web_dist +from dimos_build import bundle_native_sources, ensure_web_dist, native_files def python_is_macos_universal_binary(executable: str | None = None) -> bool: @@ -101,6 +101,9 @@ def run(self): super().run() if not getattr(self, "editable_mode", False): self._copy_relay_dist() + bundle_native_sources( + Path(__file__).parent, Path(self.build_lib) / "dimos/_native_sources.tar" + ) def _copy_relay_dist(self): src = Path(__file__).parent / "web" @@ -126,6 +129,13 @@ def _copy_relay_dist(self): class sdist(_sdist): + def make_release_tree(self, base_dir, files): + sources = [ + str(path.relative_to(Path(__file__).parent)) + for path in native_files(Path(__file__).parent) + ] + super().make_release_tree(base_dir, sorted(set(files) | set(sources))) + def run(self): ensure_web_dist(Path(__file__).parent) super().run() From 73c2fd8e9bc1973917c6eddfc6af1fee81a9aed0 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 9 Sep 2026 22:25:25 -0700 Subject: [PATCH 2/5] fix(packaging): build SDK assets during source installation --- MANIFEST.in | 2 + dimos_build.py | 36 +++++++++++++++++ installer/src/dimup/sdk.py | 49 +++++++++++++++++++++++ installer/src/dimup/test_sdk.py | 58 ++++++++++++++++++++++++++++ setup.py | 36 +++++++---------- tests/packaging/test_source_build.py | 51 ++++++++++++++++++++++++ 6 files changed, 210 insertions(+), 22 deletions(-) create mode 100644 dimos_build.py create mode 100644 installer/src/dimup/sdk.py create mode 100644 installer/src/dimup/test_sdk.py create mode 100644 tests/packaging/test_source_build.py diff --git a/MANIFEST.in b/MANIFEST.in index 3aee0bf04e..e0dcde6764 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -46,3 +46,5 @@ prune dimos/web/command-center-extension global-exclude test_*.py global-exclude conftest.py + +include dimos_build.py diff --git a/dimos_build.py b/dimos_build.py new file mode 100644 index 0000000000..8d2e30cfb1 --- /dev/null +++ b/dimos_build.py @@ -0,0 +1,36 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build-time helpers that do not import the DimOS runtime.""" + +from pathlib import Path +import shutil +import subprocess + + +def ensure_web_dist(root: Path) -> None: + """Build the web assets for Git installs and source distributions.""" + for project, artifact in (("sdk", "sdk.js"), ("cockpit", "index.html")): + source = root / "web" / project + if (source / "dist" / artifact).is_file(): + continue + deno = shutil.which("deno") + if deno is None: + raise RuntimeError( + "Deno is required to build DimOS web assets from source. " + "Run dimup setup, then retry dependency installation." + ) + subprocess.run([deno, "task", "--cwd", str(source), "build"], check=True) + if not (source / "dist" / artifact).is_file(): + raise RuntimeError(f"Deno build did not produce {source / 'dist' / artifact}") diff --git a/installer/src/dimup/sdk.py b/installer/src/dimup/sdk.py new file mode 100644 index 0000000000..f76939711b --- /dev/null +++ b/installer/src/dimup/sdk.py @@ -0,0 +1,49 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read the consumer dependency policy from the selected SDK revision.""" + +from copy import deepcopy +from typing import Any + +from dimup.process import SetupError + +SDK_URL = "https://github.com/dimensionalOS/dimos.git" + + +def consumer_policy(manifest: dict[str, Any]) -> tuple[list[str], dict[str, Any]]: + extras = sorted(set(manifest["project"]["optional-dependencies"]) - {"dds", "unitree-dds"}) + if not extras: + raise SetupError("Selected SDK has no desktop extras.") + upstream = manifest.get("tool", {}).get("uv", {}) + policy = { + key: deepcopy(upstream[key]) + for key in ( + "required-version", + "override-dependencies", + "constraint-dependencies", + "exclude-newer", + "exclude-newer-package", + "sources", + "index", + "extra-build-dependencies", + "extra-build-variables", + ) + if key in upstream + } + for name, source in policy.get("sources", {}).items(): + entries = source if isinstance(source, list) else [source] + if any("path" in entry or "workspace" in entry for entry in entries): + raise SetupError(f"Selected SDK source {name!r} requires a checkout-local dependency.") + return extras, policy diff --git a/installer/src/dimup/test_sdk.py b/installer/src/dimup/test_sdk.py new file mode 100644 index 0000000000..495b00229d --- /dev/null +++ b/installer/src/dimup/test_sdk.py @@ -0,0 +1,58 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from dimup.process import SetupError +from dimup.sdk import consumer_policy + + +def test_consumer_policy_keeps_sdk_extras_and_sources_without_dev_groups(): + manifest = { + "project": { + "optional-dependencies": { + "all": [], + "spot": [], + "learning": [], + "dds": [], + "unitree-dds": [], + } + }, + "dependency-groups": {"docs": ["sphinx"]}, + "tool": { + "uv": { + "sources": {"graspgenx": {"git": "https://example.com/grasp", "rev": "abc"}}, + "override-dependencies": ["numpy>=2"], + "default-groups": ["docs"], + } + }, + } + extras, policy = consumer_policy(manifest) + assert extras == ["all", "learning", "spot"] + assert policy == { + "sources": {"graspgenx": {"git": "https://example.com/grasp", "rev": "abc"}}, + "override-dependencies": ["numpy>=2"], + } + policy["sources"]["graspgenx"]["rev"] = "changed" + assert manifest["tool"]["uv"]["sources"]["graspgenx"]["rev"] == "abc" + + +def test_consumer_policy_rejects_nonportable_sources(): + with pytest.raises(SetupError, match="checkout-local"): + consumer_policy( + { + "project": {"optional-dependencies": {"all": []}}, + "tool": {"uv": {"sources": {"local": {"path": "../local"}}}}, + } + ) diff --git a/setup.py b/setup.py index 6eca175e1e..e2f0c441df 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,11 @@ from pybind11.setup_helpers import Pybind11Extension, build_ext from setuptools import find_packages, setup from setuptools.command.build_py import build_py as _build_py +from setuptools.command.sdist import sdist as _sdist + +# PEP 517 executes this file without adding the source root to sys.path. +sys.path.insert(0, str(Path(__file__).parent)) +from dimos_build import ensure_web_dist def python_is_macos_universal_binary(executable: str | None = None) -> bool: @@ -91,6 +96,8 @@ def find_package_modules(self, package, package_dir): ] def run(self): + if not getattr(self, "editable_mode", False): + ensure_web_dist(Path(__file__).parent) super().run() if not getattr(self, "editable_mode", False): self._copy_relay_dist() @@ -99,27 +106,6 @@ def _copy_relay_dist(self): src = Path(__file__).parent / "web" if not (src / "relay" / "main.ts").is_file(): raise RuntimeError(f"relay sources missing at {src}; refusing to build the wheel") - missing = [ - rel - for rel in ("cockpit/dist/index.html", "sdk/dist/sdk.js") - if not (src / rel).is_file() - ] - if missing: - # Wheels must carry the Cockpit and the SDK bundle: there is no - # fallback debug page, so a dist-less wheel's relay serves no UI - # (and /sdk.js only a build hint). Building a deliberate - # python-only wheel (e.g. where deno is unavailable) requires the - # explicit env-var opt-out, which covers both products. - if os.environ.get("DIMOS_ALLOW_MISSING_COCKPIT") != "1": - raise RuntimeError( - f"web dist missing ({', '.join(missing)}); run `deno task build` " - "in web/sdk and web/cockpit (or `dimos run --local-relay` from a " - "checkout builds them), or set DIMOS_ALLOW_MISSING_COCKPIT=1 to " - "build a UI-less wheel anyway" - ) - self.warn( - f"web dist missing ({', '.join(missing)}); this wheel's relay will have no UI" - ) dst = Path(self.build_lib) / RELAY_DIST_TARGET for name in RELAY_DIST_SOURCES: entry = src / name @@ -139,6 +125,12 @@ def _copy_relay_dist(self): raise RuntimeError(f"relay copy did not produce {dst / 'relay' / 'main.ts'}") +class sdist(_sdist): + def run(self): + ensure_web_dist(Path(__file__).parent) + super().run() + + extra_compile_args = [ "-O3", # Maximum optimization "-ffast-math", # Fast floating point @@ -166,5 +158,5 @@ def _copy_relay_dist(self): packages=find_packages(), package_dir={"": "."}, ext_modules=ext_modules, - cmdclass={"build_ext": build_ext, "build_py": build_py}, + cmdclass={"build_ext": build_ext, "build_py": build_py, "sdist": sdist}, ) diff --git a/tests/packaging/test_source_build.py b/tests/packaging/test_source_build.py new file mode 100644 index 0000000000..68e64bf024 --- /dev/null +++ b/tests/packaging/test_source_build.py @@ -0,0 +1,51 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import subprocess + +import pytest + +from dimos_build import ensure_web_dist + + +def test_bundled_assets_need_no_deno(tmp_path, monkeypatch): + for project, artifact in (("sdk", "sdk.js"), ("cockpit", "index.html")): + dist = tmp_path / "web" / project / "dist" + dist.mkdir(parents=True) + (dist / artifact).write_text("built") + monkeypatch.setattr("shutil.which", lambda name: None) + ensure_web_dist(tmp_path) + + +def test_missing_deno_explains_machine_setup(tmp_path, monkeypatch): + monkeypatch.setattr("shutil.which", lambda name: None) + with pytest.raises(RuntimeError, match="dimup setup"): + ensure_web_dist(tmp_path) + + +def test_source_build_produces_both_bundles(tmp_path, monkeypatch): + commands = [] + + def build(command, *, check): + commands.append(command) + project = Path(command[3]) + dist = project / "dist" + dist.mkdir(parents=True) + (dist / ("sdk.js" if project.name == "sdk" else "index.html")).write_text("built") + + monkeypatch.setattr("shutil.which", lambda name: "/bin/deno") + monkeypatch.setattr(subprocess, "run", build) + ensure_web_dist(tmp_path) + assert [Path(command[3]).name for command in commands] == ["sdk", "cockpit"] From a025256244e60ad7908ddd3bcae18a74d84faa52 Mon Sep 17 00:00:00 2001 From: cc Date: Wed, 9 Sep 2026 23:26:40 -0700 Subject: [PATCH 3/5] fix(native): build Point-LIO without PCL visualization dependencies --- .../sensors/lidar/pointlio/cpp/flake.nix | 63 ++++++++----------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/dimos/hardware/sensors/lidar/pointlio/cpp/flake.nix b/dimos/hardware/sensors/lidar/pointlio/cpp/flake.nix index 4a63b801b7..fbe269fe88 100644 --- a/dimos/hardware/sensors/lidar/pointlio/cpp/flake.nix +++ b/dimos/hardware/sensors/lidar/pointlio/cpp/flake.nix @@ -32,41 +32,32 @@ outputs = { self, nixpkgs, flake-utils, livox-sdk, dimos-lcm, pfr, fast-lio, lcm-extended, ... }: flake-utils.lib.eachDefaultSystem (system: let - # Overlay fixes for darwin-broken nixpkgs recipes in our transitive - # dep chain (pcl → vtk → pdal → tiledb → libpqxx). Each of these - # should go upstream; kept here so we can build in the meantime. - # - # Gated on isDarwin so Linux keeps binary-cache hits for the stock - # libpqxx / tiledb / pdal / vtk / pcl derivations. Applying the - # override on Linux would change their input hashes and force a - # from-source rebuild of the whole chain for no benefit. - darwinDepFixes = final: prev: - if !prev.stdenv.isDarwin then { } else { - # libpqxx: postgresqlTestHook is in nativeCheckInputs - # unconditionally and that package is marked broken on darwin. - # The list is eagerly evaluated, so simply referencing it aborts - # eval. Upstream fix is to wrap the list in - # `lib.optionals (meta.availableOn ...)`. - libpqxx = prev.libpqxx.overrideAttrs (_old: { - nativeCheckInputs = [ ]; - doCheck = false; - }); - # tiledb: darwin-only patch `generate_embedded_data_header.patch` - # targets a file that doesn't exist in tiledb 2.30.0 (the - # upstream code path was reworked and `file(ARCHIVE_CREATE ...)` - # is no longer used anywhere in the source). Filter out only - # that patch — don't drop everything, in case nixpkgs adds an - # unrelated security patch in a future bump. - tiledb = prev.tiledb.overrideAttrs (old: { - patches = builtins.filter - (p: !(prev.lib.hasSuffix "generate_embedded_data_header.patch" (toString p))) - (old.patches or [ ]); - }); - }; - pkgs = import nixpkgs { - inherit system; - overlays = [ darwinDepFixes ]; - }; + pkgs = import nixpkgs { inherit system; }; + # Point-LIO uses common/filters and includes PCL I/O headers. + # Avoid VTK's large, uncached Darwin dependency tree entirely. + pcl = pkgs.pcl.overrideAttrs (old: { + nativeBuildInputs = [ pkgs.cmake pkgs.pkg-config ]; + buildInputs = [ pkgs.eigen pkgs.boost pkgs.flann pkgs.qhull pkgs.zlib pkgs.cjson ] + ++ pkgs.lib.optionals pkgs.stdenv.cc.isClang [ pkgs.llvmPackages.openmp ]; + propagatedBuildInputs = [ pkgs.boost pkgs.flann ]; + cmakeFlags = (old.cmakeFlags or [ ]) ++ [ + "-DWITH_VTK=OFF" + "-DWITH_QT=OFF" + "-DWITH_OPENGL=OFF" + "-DBUILD_features=OFF" + "-DBUILD_ml=OFF" + "-DBUILD_segmentation=OFF" + "-DBUILD_surface=OFF" + "-DBUILD_registration=OFF" + "-DBUILD_keypoints=OFF" + "-DBUILD_tracking=OFF" + "-DBUILD_visualization=OFF" + "-DBUILD_tools=OFF" + "-DBUILD_apps=OFF" + "-DBUILD_examples=OFF" + "-DBUILD_global_tests=OFF" + ]; + }); livox-sdk2 = livox-sdk.packages.${system}.livox-sdk2; lcm = lcm-extended.packages.${system}.lcm; @@ -94,7 +85,7 @@ lcm pkgs.glib pkgs.eigen - pkgs.pcl + pcl pkgs.glog pkgs.boost pkgs.llvmPackages.openmp From d09c7577825e637ca21ada287e392af059c07380 Mon Sep 17 00:00:00 2001 From: cc Date: Thu, 10 Sep 2026 12:19:08 -0700 Subject: [PATCH 4/5] refactor(installer): isolate SDK packaging from application creation --- .github/workflows/dimup.yml | 4 +-- installer/src/dimup/sdk.py | 49 ---------------------------- installer/src/dimup/test_sdk.py | 58 --------------------------------- 3 files changed, 2 insertions(+), 109 deletions(-) delete mode 100644 installer/src/dimup/sdk.py delete mode 100644 installer/src/dimup/test_sdk.py diff --git a/.github/workflows/dimup.yml b/.github/workflows/dimup.yml index 867ab989fa..340801809b 100644 --- a/.github/workflows/dimup.yml +++ b/.github/workflows/dimup.yml @@ -23,10 +23,10 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v10.0.1 - - name: Installer tests + - name: Installer and SDK packaging tests env: PYTHONPATH: . - run: uv run --project installer --with pytest pytest --noconftest -c installer/pyproject.toml installer/src + run: uv run --project installer --with pytest --with filelock pytest --noconftest -c installer/pyproject.toml installer/src tests/packaging dimos/core/test_native_sources.py - name: Type check dimup env: MYPYPATH: installer/src diff --git a/installer/src/dimup/sdk.py b/installer/src/dimup/sdk.py deleted file mode 100644 index f76939711b..0000000000 --- a/installer/src/dimup/sdk.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Read the consumer dependency policy from the selected SDK revision.""" - -from copy import deepcopy -from typing import Any - -from dimup.process import SetupError - -SDK_URL = "https://github.com/dimensionalOS/dimos.git" - - -def consumer_policy(manifest: dict[str, Any]) -> tuple[list[str], dict[str, Any]]: - extras = sorted(set(manifest["project"]["optional-dependencies"]) - {"dds", "unitree-dds"}) - if not extras: - raise SetupError("Selected SDK has no desktop extras.") - upstream = manifest.get("tool", {}).get("uv", {}) - policy = { - key: deepcopy(upstream[key]) - for key in ( - "required-version", - "override-dependencies", - "constraint-dependencies", - "exclude-newer", - "exclude-newer-package", - "sources", - "index", - "extra-build-dependencies", - "extra-build-variables", - ) - if key in upstream - } - for name, source in policy.get("sources", {}).items(): - entries = source if isinstance(source, list) else [source] - if any("path" in entry or "workspace" in entry for entry in entries): - raise SetupError(f"Selected SDK source {name!r} requires a checkout-local dependency.") - return extras, policy diff --git a/installer/src/dimup/test_sdk.py b/installer/src/dimup/test_sdk.py deleted file mode 100644 index 495b00229d..0000000000 --- a/installer/src/dimup/test_sdk.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2026 Dimensional Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from dimup.process import SetupError -from dimup.sdk import consumer_policy - - -def test_consumer_policy_keeps_sdk_extras_and_sources_without_dev_groups(): - manifest = { - "project": { - "optional-dependencies": { - "all": [], - "spot": [], - "learning": [], - "dds": [], - "unitree-dds": [], - } - }, - "dependency-groups": {"docs": ["sphinx"]}, - "tool": { - "uv": { - "sources": {"graspgenx": {"git": "https://example.com/grasp", "rev": "abc"}}, - "override-dependencies": ["numpy>=2"], - "default-groups": ["docs"], - } - }, - } - extras, policy = consumer_policy(manifest) - assert extras == ["all", "learning", "spot"] - assert policy == { - "sources": {"graspgenx": {"git": "https://example.com/grasp", "rev": "abc"}}, - "override-dependencies": ["numpy>=2"], - } - policy["sources"]["graspgenx"]["rev"] = "changed" - assert manifest["tool"]["uv"]["sources"]["graspgenx"]["rev"] == "abc" - - -def test_consumer_policy_rejects_nonportable_sources(): - with pytest.raises(SetupError, match="checkout-local"): - consumer_policy( - { - "project": {"optional-dependencies": {"all": []}}, - "tool": {"uv": {"sources": {"local": {"path": "../local"}}}}, - } - ) From 0264bc14f05ee293a88802bda2f272994135726a Mon Sep 17 00:00:00 2001 From: cc Date: Thu, 10 Sep 2026 14:17:15 -0700 Subject: [PATCH 5/5] fix(packaging): make generated frontend assets optional --- .github/workflows/dimup.yml | 2 +- dimos_build.py | 19 ------- setup.py | 12 ++--- tests/packaging/test_source_build.py | 74 +++++++++++++++------------- 4 files changed, 46 insertions(+), 61 deletions(-) diff --git a/.github/workflows/dimup.yml b/.github/workflows/dimup.yml index 887b78e465..c76b821e51 100644 --- a/.github/workflows/dimup.yml +++ b/.github/workflows/dimup.yml @@ -34,7 +34,7 @@ jobs: - name: Installer and SDK packaging tests env: PYTHONPATH: . - run: uv run --project installer --with pytest --with filelock pytest --noconftest -c installer/pyproject.toml installer/src tests/packaging dimos/core/test_native_sources.py + run: uv run --project installer --with pytest --with filelock --with setuptools --with pybind11 pytest --noconftest -c installer/pyproject.toml installer/src tests/packaging dimos/core/test_native_sources.py - name: Type check dimup env: MYPYPATH: installer/src diff --git a/dimos_build.py b/dimos_build.py index 974b5a631b..742576e6f0 100644 --- a/dimos_build.py +++ b/dimos_build.py @@ -16,8 +16,6 @@ import os from pathlib import Path -import shutil -import subprocess import tarfile try: @@ -26,23 +24,6 @@ import tomli as tomllib -def ensure_web_dist(root: Path) -> None: - """Build the web assets for Git installs and source distributions.""" - for project, artifact in (("sdk", "sdk.js"), ("cockpit", "index.html")): - source = root / "web" / project - if (source / "dist" / artifact).is_file(): - continue - deno = shutil.which("deno") - if deno is None: - raise RuntimeError( - "Deno is required to build DimOS web assets from source. " - "Run dimup setup, then retry dependency installation." - ) - subprocess.run([deno, "task", "--cwd", str(source), "build"], check=True) - if not (source / "dist" / artifact).is_file(): - raise RuntimeError(f"Deno build did not produce {source / 'dist' / artifact}") - - def native_files(root: Path) -> list[Path]: """Collect native build roots, preserving workspace and sibling dependencies.""" workspace = tomllib.loads((root / "Cargo.toml").read_text())["workspace"] diff --git a/setup.py b/setup.py index bbc779fd97..9405badcf3 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ # PEP 517 executes this file without adding the source root to sys.path. sys.path.insert(0, str(Path(__file__).parent)) -from dimos_build import bundle_native_sources, ensure_web_dist, native_files +from dimos_build import bundle_native_sources, native_files def python_is_macos_universal_binary(executable: str | None = None) -> bool: @@ -96,8 +96,6 @@ def find_package_modules(self, package, package_dir): ] def run(self): - if not getattr(self, "editable_mode", False): - ensure_web_dist(Path(__file__).parent) super().run() if not getattr(self, "editable_mode", False): self._copy_relay_dist() @@ -112,7 +110,9 @@ def _copy_relay_dist(self): dst = Path(self.build_lib) / RELAY_DIST_TARGET for name in RELAY_DIST_SOURCES: entry = src / name - if not entry.exists(): # only the dists may be absent (env-var opt-out above) + if ( + not entry.exists() + ): # Frontend bundles are optional; Python builds never create them. continue for path in sorted(entry.rglob("*")) if entry.is_dir() else [entry]: # Filter on the path below src: matching path.parts would also @@ -136,10 +136,6 @@ def make_release_tree(self, base_dir, files): ] super().make_release_tree(base_dir, sorted(set(files) | set(sources))) - def run(self): - ensure_web_dist(Path(__file__).parent) - super().run() - extra_compile_args = [ "-O3", # Maximum optimization diff --git a/tests/packaging/test_source_build.py b/tests/packaging/test_source_build.py index 68e64bf024..04479c1868 100644 --- a/tests/packaging/test_source_build.py +++ b/tests/packaging/test_source_build.py @@ -13,39 +13,47 @@ # limitations under the License. from pathlib import Path +import runpy +import shutil import subprocess +import sys import pytest - -from dimos_build import ensure_web_dist - - -def test_bundled_assets_need_no_deno(tmp_path, monkeypatch): - for project, artifact in (("sdk", "sdk.js"), ("cockpit", "index.html")): - dist = tmp_path / "web" / project / "dist" - dist.mkdir(parents=True) - (dist / artifact).write_text("built") - monkeypatch.setattr("shutil.which", lambda name: None) - ensure_web_dist(tmp_path) - - -def test_missing_deno_explains_machine_setup(tmp_path, monkeypatch): - monkeypatch.setattr("shutil.which", lambda name: None) - with pytest.raises(RuntimeError, match="dimup setup"): - ensure_web_dist(tmp_path) - - -def test_source_build_produces_both_bundles(tmp_path, monkeypatch): - commands = [] - - def build(command, *, check): - commands.append(command) - project = Path(command[3]) - dist = project / "dist" - dist.mkdir(parents=True) - (dist / ("sdk.js" if project.name == "sdk" else "index.html")).write_text("built") - - monkeypatch.setattr("shutil.which", lambda name: "/bin/deno") - monkeypatch.setattr(subprocess, "run", build) - ensure_web_dist(tmp_path) - assert [Path(command[3]).name for command in commands] == ["sdk", "cockpit"] +from setuptools import Distribution +from setuptools.command.build_py import build_py as setuptools_build_py + + +@pytest.mark.parametrize("bundled", [False, True]) +def test_python_build_copies_optional_assets_without_frontend_tools(tmp_path, monkeypatch, bundled): + repository = Path(__file__).resolve().parents[2] + shutil.copy(repository / "setup.py", tmp_path / "setup.py") + source = tmp_path / "web" + (source / "relay").mkdir(parents=True) + (source / "relay/main.ts").write_text("relay") + (source / "deno.json").write_text("{}") + if bundled: + for project, artifact in (("sdk", "sdk.js"), ("cockpit", "index.html")): + dist = source / project / "dist" + dist.mkdir(parents=True) + (dist / artifact).write_text("prebuilt") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "path", sys.path.copy()) + monkeypatch.setattr("setuptools.setup", lambda **kwargs: None) + namespace = runpy.run_path(str(tmp_path / "setup.py")) + monkeypatch.setattr(setuptools_build_py, "run", lambda self: None) + monkeypatch.setitem( + namespace["build_py"].run.__globals__, "bundle_native_sources", lambda *args: None + ) + + def unexpected_command(*args, **kwargs): + pytest.fail("Python packaging must not invoke frontend tools") + + monkeypatch.setattr(subprocess, "run", unexpected_command) + command = namespace["build_py"](Distribution()) + command.build_lib = str(tmp_path / "build") + command.run() + packaged = tmp_path / "build/dimos/web/relay_bridge/_relay_dist" + assert (packaged / "relay/main.ts").read_text() == "relay" + assert (packaged / "cockpit/dist/index.html").exists() is bundled + assert (packaged / "sdk/dist/sdk.js").exists() is bundled + assert "run" not in namespace["sdist"].__dict__