Make the SDK installable outside its repository

.github/workflows/dimup.yml

@@ -31,10 +31,10 @@ jobs:
       - uses: astral-sh/setup-uv@v10.0.1
       - name: Install shells for configuration tests
         run: sudo apt-get update && sudo apt-get install -y zsh fish
-      - name: Installer tests
+      - name: Installer and SDK packaging tests
         env:
           PYTHONPATH: .
-        run: uv run --project installer --with pytest pytest --noconftest -c installer/pyproject.toml installer/src
+        run: uv run --project installer --with pytest --with filelock --with setuptools --with pybind11 pytest --noconftest -c installer/pyproject.toml installer/src tests/packaging dimos/core/test_native_sources.py
       - name: Type check dimup
         env:
           MYPYPATH: installer/src

MANIFEST.in

@@ -46,3 +46,5 @@ prune dimos/web/command-center-extension
 
 global-exclude test_*.py
 global-exclude conftest.py
+
+include dimos_build.py

dimos/core/native_module.py

@@ -54,12 +54,14 @@ class MyCppModule(NativeModule):
 import time
 from typing import IO, Any
 
+from filelock import FileLock
 from pydantic import Field, model_validator
 
-from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT
+from dimos.constants import DEFAULT_THREAD_JOIN_TIMEOUT, DIMOS_PROJECT_ROOT
 from dimos.core.core import rpc
 from dimos.core.global_config import global_config
 from dimos.core.module import Module, ModuleConfig
+from dimos.core.native_sources import native_source_root
 from dimos.core.transport_factory import session_config
 from dimos.protocol.service.spec import SessionConfig
 from dimos.utils.logging_config import setup_logger
@@ -120,6 +122,7 @@ class NativeModuleConfig(ModuleConfig):
     executable: str
     build_command: str | None = None
     cwd: str | None = None
+    bundled_sources: bool = False
     extra_args: list[str] = Field(default_factory=list)
     extra_env: dict[str, str] = Field(default_factory=dict)
     # Session settings for this module alone, e.g. opening it as the zenoh router
@@ -223,7 +226,13 @@ def __init__(self, **kwargs: Any) -> None:
 
         if self.config.cwd is not None and not Path(self.config.cwd).is_absolute():
             base_dir = Path(inspect.getfile(type(self))).resolve().parent
+            if self.config.bundled_sources:
+                base_dir = native_source_root() / base_dir.relative_to(DIMOS_PROJECT_ROOT)
             self.config.cwd = str(base_dir / self.config.cwd)
+            if self.config.bundled_sources:
+                self.config.extra_env.setdefault(
+                    "CARGO_TARGET_DIR", str(Path(self.config.cwd) / "target")
+                )
         if not Path(self.config.executable).is_absolute():
             # The spawn runs from the executable's own directory, so a relative
             # path has to be resolved before then or it resolves against itself.
@@ -233,7 +242,11 @@ def __init__(self, **kwargs: Any) -> None:
     @rpc
     def build(self) -> None:
         super().build()
-        self._maybe_build()
+        if self.config.bundled_sources and self.config.cwd:
+            with FileLock(str(Path(self.config.cwd) / ".build.lock")):
+                self._maybe_build()
+        else:
+            self._maybe_build()
 
     def _spawn_env(self) -> dict[str, str]:
         env = {**os.environ, **self.config.extra_env}

dimos/core/native_sources.py

@@ -0,0 +1,83 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Writable native build sources belonging to the installed SDK."""
+
+from hashlib import sha256
+import os
+from pathlib import Path
+import platform
+import shutil
+import subprocess
+import tarfile
+import tempfile
+
+from filelock import FileLock
+
+from dimos.constants import CACHE_DIR, DIMOS_PROJECT_ROOT
+
+
+def unpack_sources(bundle: Path, cache: Path) -> Path:
+    digest = sha256(bundle.read_bytes()).hexdigest()
+    root = cache / f"{digest}-{platform.system()}-{platform.machine()}"
+    cache.mkdir(parents=True, exist_ok=True)
+    with FileLock(str(root) + ".lock"):
+        if (root / ".git").is_dir():
+            return root
+        temporary = Path(tempfile.mkdtemp(prefix="native-", dir=cache))
+        try:
+            with tarfile.open(bundle) as archive:
+                archive.extractall(temporary, filter="data")
+            # Existing nested flakes use repository-relative inputs. A local
+            # snapshot lets Nix include their sibling sources without fetching
+            # another DimOS checkout or writing into site-packages.
+            (temporary / ".gitignore").write_text("target/\nresult\nresult-*\n.build.lock\n")
+            git_env = {
+                key: value for key, value in os.environ.items() if not key.startswith("GIT_")
+            }
+            git_env.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1")
+            for args in (
+                ["init", "--quiet"],
+                ["add", "."],
+                [
+                    "-c",
+                    "user.name=DimOS",
+                    "-c",
+                    "user.email=build@dimos.invalid",
+                    "-c",
+                    "core.hooksPath=/dev/null",
+                    "commit",
+                    "--quiet",
+                    "--no-gpg-sign",
+                    "-m",
+                    "Native SDK sources",
+                ],
+            ):
+                subprocess.run(
+                    ["git", *args], cwd=temporary, env=git_env, check=True, capture_output=True
+                )
+            temporary.rename(root)
+        finally:
+            if temporary.exists():
+                shutil.rmtree(temporary)
+    return root
+
+
+def native_source_root() -> Path:
+    bundle = Path(__file__).resolve().parents[1] / "_native_sources.tar"
+    if bundle.is_file():
+        return unpack_sources(bundle, CACHE_DIR / "native")
+    if (DIMOS_PROJECT_ROOT / "Cargo.toml").is_file():
+        return DIMOS_PROJECT_ROOT
+    raise FileNotFoundError("The DimOS SDK is missing its native sources. Reinstall the SDK.")

dimos/core/test_native_sources.py

@@ -0,0 +1,44 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tarfile
+
+from dimos.core.native_sources import unpack_sources
+
+
+def test_native_sources_are_reusable_writable_snapshots(tmp_path):
+    source = tmp_path / "Cargo.toml"
+    source.write_text("[workspace]\nmembers = []\n")
+    bundle = tmp_path / "sources.tar"
+    with tarfile.open(bundle, "w") as archive:
+        archive.add(source, arcname="Cargo.toml")
+    root = unpack_sources(bundle, tmp_path / "cache")
+    assert (root / "Cargo.toml").read_text() == source.read_text()
+    assert (root / ".git").is_dir()
+    (root / "target").mkdir()
+    assert unpack_sources(bundle, tmp_path / "cache") == root
+    assert (root / "target").is_dir()
+
+
+def test_source_content_selects_separate_build_roots(tmp_path):
+    source = tmp_path / "Cargo.toml"
+    bundle = tmp_path / "sources.tar"
+    roots = []
+    for version in ("one", "two"):
+        source.write_text(version)
+        with tarfile.open(bundle, "w") as archive:
+            archive.add(source, arcname="Cargo.toml")
+        roots.append(unpack_sources(bundle, tmp_path / "cache"))
+    assert roots[0] != roots[1]
+    assert (roots[0] / "Cargo.toml").read_text() == "one"

dimos/experimental/memory/rust_recorder.py

@@ -95,6 +95,7 @@ class RustRecorderConfig(NativeModuleConfig):
 
     executable: str = "result/bin/dimos-memory-recorder"
     build_command: str = "nix build -L .#dimos-memory-recorder"
+    bundled_sources: bool = True
     cwd: str = "rust"
     stdin_config: bool = True
 

dimos/hardware/sensors/camera/realsense/camera.py

@@ -34,6 +34,7 @@
 
 
 class RealSenseCameraConfig(NativeModuleConfig, DepthCameraConfig):
+    bundled_sources: bool = True
     cwd: str | None = "rust"
     executable: str = "target/release/realsense_native"
     # Own flake: librealsense2 isn't in the root shell.

dimos/hardware/sensors/lidar/fastlio2/module.py

@@ -57,6 +57,7 @@
 
 
 class FastLio2Config(NativeModuleConfig):
+    bundled_sources: bool = True
     cwd: str | None = "cpp"
     executable: str = "result/bin/fastlio2_native"
     build_command: str | None = "nix build -L .#fastlio2_native"

dimos/hardware/sensors/lidar/livox/module.py

@@ -54,6 +54,7 @@
 
 
 class Mid360Config(NativeModuleConfig):
+    bundled_sources: bool = True
     cwd: str | None = "cpp"
     executable: str = "result/bin/mid360_native"
     build_command: str | None = "nix build -L .#mid360_native"

dimos/hardware/sensors/lidar/pointlio/cpp/flake.nix

@@ -32,41 +32,32 @@
   outputs = { self, nixpkgs, flake-utils, livox-sdk, dimos-lcm, pfr, fast-lio, lcm-extended, ... }:
     flake-utils.lib.eachDefaultSystem (system:
       let
-        # Overlay fixes for darwin-broken nixpkgs recipes in our transitive
-        # dep chain (pcl → vtk → pdal → tiledb → libpqxx).  Each of these
-        # should go upstream; kept here so we can build in the meantime.
-        #
-        # Gated on isDarwin so Linux keeps binary-cache hits for the stock
-        # libpqxx / tiledb / pdal / vtk / pcl derivations.  Applying the
-        # override on Linux would change their input hashes and force a
-        # from-source rebuild of the whole chain for no benefit.
-        darwinDepFixes = final: prev:
-          if !prev.stdenv.isDarwin then { } else {
-            # libpqxx: postgresqlTestHook is in nativeCheckInputs
-            # unconditionally and that package is marked broken on darwin.
-            # The list is eagerly evaluated, so simply referencing it aborts
-            # eval.  Upstream fix is to wrap the list in
-            # `lib.optionals (meta.availableOn ...)`.
-            libpqxx = prev.libpqxx.overrideAttrs (_old: {
-              nativeCheckInputs = [ ];
-              doCheck = false;
-            });
-            # tiledb: darwin-only patch `generate_embedded_data_header.patch`
-            # targets a file that doesn't exist in tiledb 2.30.0 (the
-            # upstream code path was reworked and `file(ARCHIVE_CREATE ...)`
-            # is no longer used anywhere in the source).  Filter out only
-            # that patch — don't drop everything, in case nixpkgs adds an
-            # unrelated security patch in a future bump.
-            tiledb = prev.tiledb.overrideAttrs (old: {
-              patches = builtins.filter
-                (p: !(prev.lib.hasSuffix "generate_embedded_data_header.patch" (toString p)))
-                (old.patches or [ ]);
-            });
-          };
-        pkgs = import nixpkgs {
-          inherit system;
-          overlays = [ darwinDepFixes ];
-        };
+        pkgs = import nixpkgs { inherit system; };
+        # Point-LIO uses common/filters and includes PCL I/O headers.
+        # Avoid VTK's large, uncached Darwin dependency tree entirely.
+        pcl = pkgs.pcl.overrideAttrs (old: {
+          nativeBuildInputs = [ pkgs.cmake pkgs.pkg-config ];
+          buildInputs = [ pkgs.eigen pkgs.boost pkgs.flann pkgs.qhull pkgs.zlib pkgs.cjson ]
+            ++ pkgs.lib.optionals pkgs.stdenv.cc.isClang [ pkgs.llvmPackages.openmp ];
+          propagatedBuildInputs = [ pkgs.boost pkgs.flann ];
+          cmakeFlags = (old.cmakeFlags or [ ]) ++ [
+            "-DWITH_VTK=OFF"
+            "-DWITH_QT=OFF"
+            "-DWITH_OPENGL=OFF"
+            "-DBUILD_features=OFF"
+            "-DBUILD_ml=OFF"
+            "-DBUILD_segmentation=OFF"
+            "-DBUILD_surface=OFF"
+            "-DBUILD_registration=OFF"
+            "-DBUILD_keypoints=OFF"
+            "-DBUILD_tracking=OFF"
+            "-DBUILD_visualization=OFF"
+            "-DBUILD_tools=OFF"
+            "-DBUILD_apps=OFF"
+            "-DBUILD_examples=OFF"
+            "-DBUILD_global_tests=OFF"
+          ];
+        });
         livox-sdk2 = livox-sdk.packages.${system}.livox-sdk2;
         lcm = lcm-extended.packages.${system}.lcm;
 
@@ -94,7 +85,7 @@
             lcm
             pkgs.glib
             pkgs.eigen
-            pkgs.pcl
+            pcl
             pkgs.glog
             pkgs.boost
             pkgs.llvmPackages.openmp

dimos/hardware/sensors/lidar/pointlio/module.py

@@ -68,6 +68,7 @@
 
 
 class PointLioConfig(NativeModuleConfig):
+    bundled_sources: bool = True
     cwd: str | None = "cpp"
     executable: str = "result/bin/pointlio_native"
     build_command: str | None = "nix build -L .#pointlio_native"

dimos/hardware/sensors/lidar/virtual_mid360/module.py

@@ -36,7 +36,6 @@
 
 from pydantic import Field
 
-from dimos.constants import DIMOS_PROJECT_ROOT
 from dimos.core.core import rpc
 from dimos.core.native_module import NativeModule, NativeModuleConfig
 from dimos.utils.logging_config import setup_logger
@@ -55,9 +54,10 @@
 
 
 class VirtualMid360Config(NativeModuleConfig):
+    bundled_sources: bool = True
     cwd: str | None = "."
     # The crate is a workspace member, so cargo builds into the repo-root target dir.
-    executable: str = str(DIMOS_PROJECT_ROOT / "target" / "release" / "virtual_mid360")
+    executable: str = "target/release/virtual_mid360"
     build_command: str | None = "cargo build --release"
     # The rust binary reads its config as a JSON object on stdin (required).
     stdin_config: bool = True

dimos/mapping/dim_slam/dim_slam.py

@@ -120,6 +120,7 @@ class SourceConfig(BaseModel):
 
 
 class DimSlamConfig(NativeModuleConfig):
+    bundled_sources: bool = True
     cwd: str | None = "rust"
     executable: str = "result/bin/dim_slam"
     build_command: str | None = Field(

dimos/mapping/dim_slam/rust/flake.nix

@@ -7,15 +7,11 @@
     cu-vslam-rs.url = "github:jeff-hykin/cu_vslam_rs";
     cu-vslam-rs.inputs.nixpkgs.follows = "nixpkgs";
     cu-vslam-rs.inputs.flake-utils.follows = "flake-utils";
-    # Relative git+file: will be deprecated (nix#12281) but there's no
-    # viable alternative for reaching local path deps outside the flake dir currently
-    # presumably an alternative will be added before this is removed.
-    dimos-repo = { url = "git+file:../../../.."; flake = false; };
     crate2nix.url = "github:nix-community/crate2nix";
     crate2nix.inputs.nixpkgs.follows = "nixpkgs";
   };
 
-  outputs = { self, nixpkgs, flake-utils, cu-vslam-rs, dimos-repo, crate2nix }:
+  outputs = { self, nixpkgs, flake-utils, cu-vslam-rs, crate2nix }:
     # Not eachDefaultSystem: nixpkgs 26.11 dropped x86_64-darwin, and merely naming
     # it is an eval error.
     flake-utils.lib.eachSystem [ "aarch64-darwin" "aarch64-linux" "x86_64-linux" ] (system:
@@ -42,8 +38,8 @@
           cp ${./build.rs} $out/dimos/mapping/dim_slam/rust/build.rs
 
           mkdir -p $out/native/rust
-          cp -r ${dimos-repo}/native/rust/dimos-module $out/native/rust/dimos-module
-          cp -r ${dimos-repo}/native/rust/dimos-module-macros $out/native/rust/dimos-module-macros
+          cp -r ${../../../../native/rust/dimos-module} $out/native/rust/dimos-module
+          cp -r ${../../../../native/rust/dimos-module-macros} $out/native/rust/dimos-module-macros
         '';
 
         # One derivation per crate rather than one vendored blob, so a dependency bump

dimos/mapping/ray_tracing/module.py

@@ -17,7 +17,6 @@
 
 from typing import TYPE_CHECKING
 
-from dimos.constants import DIMOS_PROJECT_ROOT
 from dimos.core.native_module import NativeModule, NativeModuleConfig
 from dimos.core.stream import In, Out
 from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped
@@ -32,9 +31,10 @@
 
 
 class RayTracingVoxelMapConfig(NativeModuleConfig):
+    bundled_sources: bool = True
     cwd: str | None = "rust"
     # The crate is a workspace member, so cargo builds into the repo-root target dir.
-    executable: str = str(DIMOS_PROJECT_ROOT / "target" / "release" / "voxel_ray_tracing")
+    executable: str = "target/release/voxel_ray_tracing"
     build_command: str | None = "cargo build --release"
     stdin_config: bool = True