diff --git a/ccflow/__init__.py b/ccflow/__init__.py index 3cc404cf..1434a906 100644 --- a/ccflow/__init__.py +++ b/ccflow/__init__.py @@ -5,6 +5,7 @@ # which, in turn, import `ccflow`). from .exttypes import * # noqa: I001 +from . import config as config from .base import * from .compose import * from .callable import * diff --git a/ccflow/base.py b/ccflow/base.py index 92652477..cb73ddce 100644 --- a/ccflow/base.py +++ b/ccflow/base.py @@ -702,14 +702,14 @@ def create_config_from_path( Returns: The instance of the model registry, with the configs loaded. """ - import hydra # Heavy import, only import if used. + from .config import compose, initialize_config_dir overrides = overrides or [] path = pathlib.Path(path).absolute() # Hydra requires absolute paths if not path.parent.exists(): raise OSError(f"Path does not exist: {path.parent}") - with hydra.initialize_config_dir(version_base=version_base, config_dir=str(path.parent)): - cfg = hydra.compose(config_name=path.name, overrides=overrides) + with initialize_config_dir(version_base=version_base, config_dir=str(path.parent)): + cfg = compose(config_name=path.name, overrides=overrides) return cfg def load_config_from_path( @@ -888,7 +888,7 @@ def _add_pending(self, name: str, cfg: Any, overwrite: bool, lookup_registries: self._pending_lookup_registries[name] = list(lookup_registries) def _materialize(self, name: str) -> BaseModel: - from hydra.utils import instantiate + from .config import instantiate key = (id(self), name) stack = _LAZY_LOADING_STACK.get() @@ -1112,20 +1112,21 @@ def load_config( self, cfg: "DictConfig", registry: ModelRegistry, skip_exceptions: bool = False, resolve_from: ModelRegistry | None = None ) -> ModelRegistry: """Load from OmegaConf DictConfig that follows hydra conventions.""" + from .config import InstantiationException, instantiate + # Here we use hydra's 'instantiate' to instantiate models, # because it provides a standard way to resolve the class name # that's being constructed, through the "_target_" field. # This also allows for nested attributes on the model itself to # be constructed, even if they are not themselves of BaseModel type, # or if they are of a specific subclass of the parent. - from hydra.errors import InstantiationException - from hydra.utils import instantiate - from omegaconf import OmegaConf, UnsupportedValueType - if resolve_from is not None and resolve_from is not registry: initial_chain = [resolve_from, registry] else: initial_chain = [registry] + + from omegaconf import OmegaConf, UnsupportedValueType + models_to_register = self._make_subregistries(cfg, initial_chain) while True: unresolved_models = [] diff --git a/ccflow/config.py b/ccflow/config.py new file mode 100644 index 00000000..9af1fb1e --- /dev/null +++ b/ccflow/config.py @@ -0,0 +1,67 @@ +"""Expose the selected configuration backend through one API. + +Hydra is preferred when available, otherwise Lerna is used. Set ``CCFLOW_CONFIG_BACKEND=lerna`` to require Lerna instead. +The backend is loaded on first API access, so importing ccflow does not require either package. +""" + +import importlib +import os +from functools import cache +from types import ModuleType +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from hydra import compose, initialize, initialize_config_dir, main + from hydra.core.hydra_config import HydraConfig + from hydra.errors import InstantiationException + from hydra.utils import instantiate + + CONFIG_FRAMEWORK: str + +__all__ = ( + "CONFIG_FRAMEWORK", + "HydraConfig", + "InstantiationException", + "compose", + "initialize", + "initialize_config_dir", + "instantiate", + "main", +) + +_ENV_VAR = "CCFLOW_CONFIG_BACKEND" +_SUBMODULES = { + "DefaultsList": "_internal.defaults_list", + "GlobalHydra": "core.global_hydra", + "HydraConfig": "core.hydra_config", + "InstantiationException": "errors", + "ObjectType": "core.object_type", + "RunMode": "types", + "instantiate": "utils", +} + + +@cache +def _get_framework() -> ModuleType: + requested = os.environ.get(_ENV_VAR, "").strip().lower() + frameworks = ("lerna",) if requested == "lerna" else ("hydra", "lerna") + for name in frameworks: + try: + return importlib.import_module(name) + except ModuleNotFoundError as error: + if error.name != name: + raise + if requested == "lerna": + raise ImportError(f"{_ENV_VAR}=lerna requires the 'lerna' package") + raise ImportError("Configuration requires 'hydra-core' or 'lerna'; install one to use ccflow.config") + + +def __getattr__(name: str): + if name == "CONFIG_FRAMEWORK": + return _get_framework().__name__ + if name in ("main", "compose", "initialize", "initialize_config_dir"): + return getattr(_get_framework(), name) + if name in _SUBMODULES: + module = importlib.import_module(f"{_get_framework().__name__}.{_SUBMODULES[name]}") + return getattr(module, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/ccflow/examples/calculator/__main__.py b/ccflow/examples/calculator/__main__.py index 19210cfa..89181aa7 100644 --- a/ccflow/examples/calculator/__main__.py +++ b/ccflow/examples/calculator/__main__.py @@ -1,11 +1,10 @@ -import hydra - +from ccflow import config from ccflow.utils.hydra import cfg_run __all__ = ("main",) -@hydra.main(config_path="config", config_name="base", version_base=None) +@config.main(config_path="config", config_name="base", version_base=None) def main(cfg): cfg_run(cfg) diff --git a/ccflow/examples/etl/__main__.py b/ccflow/examples/etl/__main__.py index 73f13c9f..b280ae33 100644 --- a/ccflow/examples/etl/__main__.py +++ b/ccflow/examples/etl/__main__.py @@ -1,11 +1,10 @@ -import hydra - +from ccflow import config from ccflow.utils.hydra import cfg_run __all__ = ("main",) -@hydra.main(config_path="config", config_name="base", version_base=None) +@config.main(config_path="config", config_name="base", version_base=None) def main(cfg): cfg_run(cfg) diff --git a/ccflow/tests/config_user/sample2.yml b/ccflow/tests/config_user/sample2.yml deleted file mode 100644 index 9a3e99e7..00000000 --- a/ccflow/tests/config_user/sample2.yml +++ /dev/null @@ -1,6 +0,0 @@ -user_bar: - _target_: ccflow.tests.test_base_registry.MyNestedModel - x: foo - y: # Note that when type is defined on parent model, no need to specify _target_ - a: test2 - b: 2.0 \ No newline at end of file diff --git a/ccflow/tests/enums/test_enums.py b/ccflow/tests/enums/test_enums.py index 1ade6a6d..22343185 100644 --- a/ccflow/tests/enums/test_enums.py +++ b/ccflow/tests/enums/test_enums.py @@ -12,11 +12,18 @@ def auto(): class TestEnum(TestCase): + def setUp(self) -> None: + self._csp_modules = {name: module for name, module in sys.modules.items() if name == "csp" or name.startswith("csp.")} + def tearDown(self) -> None: # Because test_init_parent and test_init_parent_csp muck around with imports # Make sure we always rest the imports at the end of each test so that other # tests are unaffected os.environ.pop("CCFLOW_NO_CSP", None) + for name in list(sys.modules): + if name == "csp" or name.startswith("csp."): + sys.modules.pop(name) + sys.modules.update(self._csp_modules) importlib.invalidate_caches() import ccflow.enums diff --git a/ccflow/tests/examples/ai/test_config.py b/ccflow/tests/examples/ai/test_config.py index 9ee56c9b..8b94a3d2 100644 --- a/ccflow/tests/examples/ai/test_config.py +++ b/ccflow/tests/examples/ai/test_config.py @@ -38,7 +38,7 @@ def _options(group: str) -> list[str]: def _registry(*overrides): - return load_config(overrides=["checks=off", *overrides]) + return load_config(overrides=["checks='off'", *overrides]) class RegistryTestCase(TestCase): @@ -70,7 +70,7 @@ def test_every_group_offers_a_choice(self): def test_every_option_composes_and_instantiates(self): for group, option in ALL_OPTIONS: with self.subTest(group=group, option=option): - self.assertIsNotNone(_registry(f"{group}={option}")["/task"]) + self.assertIsNotNone(_registry(f"{group}={option!r}")["/task"]) class TestTasks(RegistryTestCase): @@ -177,9 +177,9 @@ def test_checks_are_enforced_by_default(self): def test_the_concise_agents_variant_inherits_every_session(self): # `load_config` mutates the one root registry, so each value must be read before reloading. - default = load_config(overrides=["checks=off", "agents=poem"])["/task"].arbiter.session + default = load_config(overrides=["checks='off'", "agents=poem"])["/task"].arbiter.session default_command, contract = default.command, default.output_contract - concise = load_config(overrides=["checks=off", "agents=poem_concise"])["/task"].arbiter.session + concise = load_config(overrides=["checks='off'", "agents=poem_concise"])["/task"].arbiter.session self.assertNotEqual(concise.command, default_command) self.assertEqual(concise.output_contract, contract) diff --git a/ccflow/tests/test_base_registry.py b/ccflow/tests/test_base_registry.py index 0d20b399..8e81135f 100644 --- a/ccflow/tests/test_base_registry.py +++ b/ccflow/tests/test_base_registry.py @@ -9,13 +9,13 @@ from unittest import TestCase, mock import pytest -from hydra.errors import InstantiationException from omegaconf import OmegaConf from omegaconf.errors import InterpolationKeyError from pydantic import ConfigDict, Field from ccflow import BaseModel, LazyRegistry, ModelRegistry, RegistryLookupContext, RootModelRegistry, model_alias from ccflow.base import RegistryKeyError, resolve_str +from ccflow.config import InstantiationException class MyTestModel(BaseModel): @@ -461,6 +461,25 @@ def test_load_config(self): r.load_config(cfg, overwrite=True) self.assertEqual(r["foo"], m) + def test_load_config_uses_selected_framework(self): + from ccflow import config + + cfg = OmegaConf.create( + { + "foo": { + "_target_": "ccflow.tests.test_base_registry.MyTestModel", + "a": "test", + "b": 0.0, + } + } + ) + registry = ModelRegistry(name="test") + + with mock.patch("ccflow.config.instantiate", wraps=config.instantiate) as instantiate: + registry.load_config(cfg) + + instantiate.assert_called_once() + def test_load_config_with_function(self): cfg = OmegaConf.create( { @@ -754,6 +773,17 @@ def test_defers_models_until_access(self): self.assertIs(root["/lazy/group/source"], source) self.assertEqual(LazyTestModel.constructions, 1) + def test_materialization_uses_selected_framework(self): + from ccflow import config + + root = self._load_registry() + + with mock.patch("ccflow.config.instantiate", wraps=config.instantiate) as instantiate: + source = root["/lazy/group/source"] + + self.assertIsInstance(source, LazyTestModel) + instantiate.assert_called_once() + def test_materializes_dependency_closure(self): root = self._load_registry() @@ -870,7 +900,9 @@ def test_cross_registry_cycle_reports_full_path(self): left["model"] def test_concurrent_cross_registry_cycle_does_not_deadlock(self): - from hydra.utils import instantiate as hydra_instantiate + from ccflow import config + + selected_instantiate = config.instantiate root = ModelRegistry.root() left = LazyRegistry(name="left") @@ -890,10 +922,10 @@ def synchronized_instantiate(*args, **kwargs): if not getattr(local, "started", False): local.started = True barrier.wait(timeout=2) - return hydra_instantiate(*args, **kwargs) + return selected_instantiate(*args, **kwargs) with ( - mock.patch("hydra.utils.instantiate", side_effect=synchronized_instantiate), + mock.patch("ccflow.config.instantiate", side_effect=synchronized_instantiate), concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor, ): futures = [executor.submit(registry.__getitem__, "model") for registry in (left, right)] @@ -979,6 +1011,7 @@ def test_pending_config_is_read_only_copy(self): self.assertEqual(registry["pending"].a, "pending") def test_recursive_hydra_instantiation_is_rejected(self): + from hydra.errors import InstantiationException as HydraInstantiationException from hydra.utils import instantiate cfg = OmegaConf.create( @@ -993,7 +1026,7 @@ def test_recursive_hydra_instantiation_is_rejected(self): } ) - with self.assertRaisesRegex(InstantiationException, "Set '_recursive_: false'"): + with self.assertRaisesRegex(HydraInstantiationException, "Set '_recursive_: false'"): instantiate(cfg, _convert_="all") def test_resolve_from_applies_to_direct_pending_entry(self): diff --git a/ccflow/tests/test_config.py b/ccflow/tests/test_config.py new file mode 100644 index 00000000..559e1cd7 --- /dev/null +++ b/ccflow/tests/test_config.py @@ -0,0 +1,258 @@ +import json +import os +import subprocess +import sys + +import pytest + + +def _config_framework( + env_value: str | None, + *, + hide_hydra: bool = False, + hide_lerna: bool = False, + hide_omegaconf: bool = False, +) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + if env_value is None: + env.pop("CCFLOW_CONFIG_BACKEND", None) + else: + env["CCFLOW_CONFIG_BACKEND"] = env_value + hidden_modules = [] + if hide_hydra: + hidden_modules.append("hydra") + if hide_lerna: + hidden_modules.append("lerna") + if hide_omegaconf: + hidden_modules.append("omegaconf") + hide_modules_statement = "".join(f'sys.modules["{name}"] = None;' for name in hidden_modules) + return subprocess.run( + [ + sys.executable, + "-c", + f"import sys; {hide_modules_statement} from ccflow import config; print(config.CONFIG_FRAMEWORK)", + ], + capture_output=True, + text=True, + env=env, + check=False, + ) + + +def _compose_config(config_dir, framework: str, overrides: list[str] | None = None, *, hide_hydra: bool = False) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env["CCFLOW_CONFIG_BACKEND"] = framework + return subprocess.run( + [ + sys.executable, + "-c", + ( + ("import sys; sys.modules['hydra'] = None; " if hide_hydra else "") + "from ccflow import config; " + f"context = config.initialize_config_dir(config_dir={str(config_dir)!r}, version_base=None); " + f"context.__enter__(); cfg = config.compose(config_name='config', overrides={overrides or []!r}); " + "print(cfg); context.__exit__(None, None, None)" + ), + ], + capture_output=True, + text=True, + env=env, + check=False, + ) + + +@pytest.mark.parametrize("env_value", [None, "hydra", "hydraa"]) +def test_hydra_is_preferred_when_available(env_value): + pytest.importorskip("hydra") + pytest.importorskip("lerna") + + result = _config_framework(env_value) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "hydra" + + +def test_hydra_can_be_selected_explicitly(): + pytest.importorskip("hydra") + + result = _config_framework("hydra") + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "hydra" + + +@pytest.mark.parametrize("env_value", ["lerna", " LERNA "]) +def test_lerna_can_be_selected_explicitly(env_value): + pytest.importorskip("lerna") + + result = _config_framework(env_value) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "lerna" + + +@pytest.mark.parametrize("env_value", [None, "hydra", "hydraa"]) +def test_lerna_is_used_when_hydra_is_unavailable(env_value): + pytest.importorskip("lerna") + + result = _config_framework(env_value, hide_hydra=True) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "lerna" + + +def test_hydra_is_used_when_lerna_is_unavailable(): + pytest.importorskip("hydra") + + result = _config_framework(None, hide_lerna=True) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "hydra" + + +def test_explicit_unavailable_lerna_fails_clearly(): + result = _config_framework("lerna", hide_lerna=True) + + assert result.returncode != 0 + assert "CCFLOW_CONFIG_BACKEND=lerna requires the 'lerna' package" in result.stderr + + +@pytest.mark.parametrize("env_value", [None, "hydra", "hydraa"]) +def test_missing_frameworks_fail_clearly(env_value): + result = _config_framework(env_value, hide_hydra=True, hide_lerna=True) + + assert result.returncode != 0 + assert "requires 'hydra-core' or 'lerna'" in result.stderr + + +def test_missing_backend_dependency_is_not_hidden_by_fallback(): + pytest.importorskip("hydra") + + result = _config_framework(None, hide_omegaconf=True) + + assert result.returncode != 0 + assert "ModuleNotFoundError" in result.stderr + assert "omegaconf" in result.stderr + + +def test_public_api_used_by_downstream_applications(): + from ccflow import config + + assert config.CONFIG_FRAMEWORK in ("hydra", "lerna") + assert callable(config.main) + assert callable(config.compose) + assert callable(config.initialize) + assert callable(config.initialize_config_dir) + assert callable(config.instantiate) + assert hasattr(config.HydraConfig, "get") + assert issubclass(config.InstantiationException, Exception) + + +def test_lerna_fallback_can_compose_without_hydra(tmp_path): + pytest.importorskip("lerna") + (tmp_path / "config.yaml").write_text("value: 42\n") + + result = _compose_config(tmp_path, "hydra", hide_hydra=True) + + assert result.returncode == 0, result.stderr + assert "'value': 42" in result.stdout + + +def test_lerna_patch_is_available_through_facade(tmp_path): + pytest.importorskip("hydra") + group = tmp_path / "group" + group.mkdir() + (group / "base.yaml").write_text("drop_me: true\nkeep: 42\n") + (tmp_path / "config.yaml").write_text( + """defaults: + - group/base@_here_ + - _self_ + - _patch_: + - ~drop_me +""" + ) + + lerna_result = _compose_config(tmp_path, "lerna") + hydra_result = _compose_config(tmp_path, "hydra") + + assert lerna_result.returncode == 0, lerna_result.stderr + assert "drop_me" not in lerna_result.stdout + assert "'keep': 42" in lerna_result.stdout + assert hydra_result.returncode != 0 + assert "Could not load '_patch_/~drop_me'" in hydra_result.stderr + + +def test_lerna_list_operations_are_available_through_facade(tmp_path): + pytest.importorskip("hydra") + (tmp_path / "config.yaml").write_text("tags: [one, two]\n") + + lerna_result = _compose_config(tmp_path, "lerna", ["tags=append(three)"]) + hydra_result = _compose_config(tmp_path, "hydra", ["tags=append(three)"]) + + assert lerna_result.returncode == 0, lerna_result.stderr + assert "one" in lerna_result.stdout + assert "two" in lerna_result.stdout + assert "three" in lerna_result.stdout + assert hydra_result.returncode != 0 + assert "Unknown function 'append'" in hydra_result.stderr + + +def _explain_lerna_config(config_dir, overrides: list[str] | None = None, *, structured_config: dict | None = None) -> dict: + pytest.importorskip("lerna") + setup = "" + if structured_config is not None: + setup = f"from lerna.core.config_store import ConfigStore; ConfigStore.instance().store(name='config', node={structured_config!r}); " + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import json; from ccflow.utils.hydra import load_config; " + + setup + + f"result = load_config(root_config_dir={str(config_dir)!r}, root_config_name='config', " + f"config_dir='', overrides={overrides or []!r}, debug=True); " + "print(json.dumps(result.merge()))" + ), + ], + capture_output=True, + text=True, + env={**os.environ, "CCFLOW_CONFIG_BACKEND": "lerna"}, + check=False, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +def test_lerna_debug_supports_list_operations(tmp_path): + (tmp_path / "config.yaml").write_text("tags: [one, two]\n") + + explained = _explain_lerna_config(tmp_path, ["tags=append(three)"]) + + assert explained["tags"]["value_raw"] == ["one", "two", "three"] + assert explained["tags"]["value_interp"] == ["one", "two", "three"] + + +def test_lerna_debug_preserves_yaml_scalar_types(tmp_path): + (tmp_path / "config.yaml").write_text("words: [off, on, yes, no]\n") + + explained = _explain_lerna_config(tmp_path) + + assert explained["words"]["value_raw"] == ["off", "on", "yes", "no"] + assert explained["words"]["value_interp"] == ["off", "on", "yes", "no"] + + +def test_lerna_debug_tracks_package_sources(tmp_path): + (tmp_path / "config.yaml").write_text( + "defaults:\n - function/add@from_package\n - _self_\nhydra:\n searchpath:\n - pkg://ccflow.examples.calculator.config\n" + ) + + explained = _explain_lerna_config(tmp_path) + + assert explained["from_package"]["offset"]["value_raw"] == 0.0 + assert explained["from_package"]["offset"]["source_file"] == "pkg://ccflow.examples.calculator.config/function/add.yaml" + + +def test_lerna_debug_tracks_integer_mapping_keys(tmp_path): + explained = _explain_lerna_config(tmp_path, structured_config={"codes": {200: "ok"}}) + + assert explained["codes"]["200"]["value_raw"] == "ok" + assert explained["codes"]["200"]["source_file"] == "structured://config.yaml" diff --git a/ccflow/tests/test_import.py b/ccflow/tests/test_import.py index af00bfbc..4f4dd0d0 100644 --- a/ccflow/tests/test_import.py +++ b/ccflow/tests/test_import.py @@ -27,6 +27,8 @@ def find_spec(self, name, path=None, target=None): import ccflow + assert ccflow.config.__name__ == "ccflow.config" + registry = ccflow.ModelRegistry(name="test") registry.add("model", ccflow.BaseModel()) assert isinstance(registry["model"], ccflow.BaseModel) @@ -38,14 +40,14 @@ def find_spec(self, name, path=None, target=None): for load in (lambda: lazy["model"], lambda: registry.load_config({"m": {"_target_": "ccflow.BaseModel"}})): try: load() - except ModuleNotFoundError as e: - assert e.name == "hydra", e + except ImportError as e: + assert "requires" in str(e) and "lerna" in str(e), e else: - raise AssertionError("expected hydra to be required") + raise AssertionError("expected a config framework to be required") try: import ccflow.utils.hydra # noqa: F401 - except ModuleNotFoundError: + except ImportError: pass else: raise AssertionError("ccflow.utils.hydra imported without hydra") diff --git a/ccflow/tests/utils/test_hydra.py b/ccflow/tests/utils/test_hydra.py index 1b2f59b5..888e2d55 100644 --- a/ccflow/tests/utils/test_hydra.py +++ b/ccflow/tests/utils/test_hydra.py @@ -4,8 +4,8 @@ from unittest.mock import MagicMock import pytest -from hydra import compose, initialize +from ccflow.config import compose, initialize from ccflow.utils.hydra import ( add_hydra_config_args, add_panel_server_args, @@ -265,14 +265,16 @@ def test_config_dir_with_overrides(basepath): assert "user_foo" in result.cfg["config_user"] -def test_config_name_yml_not_yaml(basepath): +def test_config_name_yml_not_yaml(basepath, tmp_path): root_config_dir = str(Path(__file__).resolve().parent.parent / "config") - config_dir = str(Path(__file__).resolve().parent.parent / "config_user") + config_dir = tmp_path / "config_user" + config_dir.mkdir() + (config_dir / "sample2.yml").write_text("foo: bar") with pytest.raises(ValueError): load_config( root_config_dir=root_config_dir, root_config_name="conf", - config_dir=config_dir, + config_dir=str(config_dir), config_name="sample2", basepath=basepath, ) @@ -338,7 +340,7 @@ def test_debug(basepath): assert "hydra/job_logging" in result.group_options assert len(result.group_options["hydra/job_logging"]) > 1 assert "config_user" in result.group_options - assert result.group_options["config_user"] == ["sample"] + assert "sample" in result.group_options["config_user"] # Arguable whether these should be here assert "conf_out_of_order" in result.group_options[""] @@ -347,7 +349,7 @@ def test_debug(basepath): assert merged assert "foo" in merged assert "config_user" in merged - assert merged["config_user"]["__options__"] == ["sample"] + assert "sample" in merged["config_user"]["__options__"] assert merged["config_user"]["__parent__"] == "conf" # Maybe this should be a path to a file assert merged["config_user"]["__selected__"] == "sample" assert "user_foo" in merged["config_user"] diff --git a/ccflow/utils/hydra.py b/ccflow/utils/hydra.py index 946b98e5..5f252b10 100644 --- a/ccflow/utils/hydra.py +++ b/ccflow/utils/hydra.py @@ -9,11 +9,11 @@ from textwrap import dedent from typing import Any -from hydra._internal.defaults_list import DefaultsList from omegaconf import DictConfig, ListConfig, OmegaConf from ..base import ModelRegistry from ..callable import FlowOptions, FlowOptionsOverride +from ..config import CONFIG_FRAMEWORK, DefaultsList, GlobalHydra, ObjectType, RunMode, compose, initialize_config_dir _log = getLogger(__name__) @@ -105,14 +105,30 @@ def _dict_add_source(d, source): return d +def _dict_add_provenance(data, composition, prefix=""): + result = {} + for key, value in data.items(): + if isinstance(key, int) and prefix: + path = f"{prefix}[{key}]" + else: + path = f"{prefix}.{key}" if prefix else str(key) + if isinstance(value, dict): + result[key] = _dict_add_provenance(value, composition, path) + else: + provenance = composition.provenance(path) + result[key] = { + "value_raw": value, + "source_file": provenance.source_identifier.removeprefix("file://") if provenance is not None else "unknown", + } + return result + + def _find_group_options(config_loader, path, config_name, overrides, results): """This function finds the names of all the options in the config group hierarchy. Note that it will pick up config files that are not intended to be used as config group options, but that exist to provide common config options to other files in the group (i.e. to default) """ - from hydra.core.object_type import ObjectType - groups = config_loader.get_group_options(path, ObjectType.GROUP, config_name, overrides) options = config_loader.get_group_options(path, ObjectType.CONFIG, config_name, overrides) if options: @@ -179,11 +195,6 @@ def load_config( basepath: The base path to start searching for the `config_dir`. This is useful when you want to load from an absolute (rather than relative) path. debug: (Experimental) Whether to enable debug mode. This will return more information about the configs on ConfigLoadResult. """ - # Heavy import, only import if used - import os - - from hydra import compose, initialize_config_dir - if return_hydra_config and debug: raise ValueError("Cannot return hydra config and debug=True at the same time. Please set return_hydra_config=False.") @@ -201,26 +212,32 @@ def load_config( else: overrides = [*overrides.copy(), f"hydra.searchpath=[{','.join(searchpaths)}]"] - cfg = compose(config_name=root_config_name, overrides=overrides, return_hydra_config=return_hydra_config) + composition = None + if debug and CONFIG_FRAMEWORK == "lerna": + from lerna import compose_with_provenance + + composition = compose_with_provenance(config_name=root_config_name, overrides=overrides) + cfg = composition.config + else: + cfg = compose(config_name=root_config_name, overrides=overrides, return_hydra_config=return_hydra_config) result = ConfigLoadResult(root_config_dir=root_config_dir, root_config_name=root_config_name, cfg=cfg) if debug: - import yaml - from hydra.core.global_hydra import GlobalHydra - from hydra.types import RunMode - - # To track the source file for each config value, we need to monkey patch the yaml loader - original_yaml_load = yaml.load - try: - - def yaml_load(*args, **kwargs): - res = original_yaml_load(*args, **kwargs) - return _dict_add_source(res, args[0].name) - - yaml.load = yaml_load - # We can't load the hydra config after monkey patching yaml loading, so skip that step - result.cfg_sources = compose(config_name=root_config_name, overrides=overrides, return_hydra_config=False) - finally: - yaml.load = original_yaml_load + if composition is not None: + result.cfg_sources = OmegaConf.create(_dict_add_provenance(OmegaConf.to_container(cfg, resolve=False), composition)) + else: + import yaml + + original_yaml_load = yaml.load + try: + + def yaml_load(*args, **kwargs): + res = original_yaml_load(*args, **kwargs) + return _dict_add_source(res, args[0].name) + + yaml.load = yaml_load + result.cfg_sources = compose(config_name=root_config_name, overrides=overrides, return_hydra_config=False) + finally: + yaml.load = original_yaml_load config_loader = GlobalHydra.instance().config_loader() # Load defaults list using the standard hydra function diff --git a/pyproject.toml b/pyproject.toml index 94a7d87e..22620304 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ full = [ "duckdb", "hydra-core", "IPython", + "lerna>2.1.0", "opentelemetry-api", "pandas", "panel", @@ -101,6 +102,7 @@ develop = [ "duckdb", "hydra-core", "IPython", + "lerna>2.1.0", "pandas", "panel", "panel_material_ui", @@ -131,6 +133,7 @@ test = [ "beautifulsoup4", "httpx2", "hydra-core", + "lerna>2.1.0", "opentelemetry-api", "pandas", "pytest",