Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ccflow/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
# which, in turn, import `ccflow`).
from .exttypes import * # noqa: I001

from . import config as config
from .base import *
from .compose import *
from .callable import *
Expand Down
17 changes: 9 additions & 8 deletions ccflow/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -702,14 +702,14 @@ def create_config_from_path(
Returns:
The instance of the model registry, with the configs loaded.
"""
import hydra # Heavy import, only import if used.
from .config import compose, initialize_config_dir

overrides = overrides or []
path = pathlib.Path(path).absolute() # Hydra requires absolute paths
if not path.parent.exists():
raise OSError(f"Path does not exist: {path.parent}")
with hydra.initialize_config_dir(version_base=version_base, config_dir=str(path.parent)):
cfg = hydra.compose(config_name=path.name, overrides=overrides)
with initialize_config_dir(version_base=version_base, config_dir=str(path.parent)):
cfg = compose(config_name=path.name, overrides=overrides)
return cfg

def load_config_from_path(
Expand Down Expand Up @@ -888,7 +888,7 @@ def _add_pending(self, name: str, cfg: Any, overwrite: bool, lookup_registries:
self._pending_lookup_registries[name] = list(lookup_registries)

def _materialize(self, name: str) -> BaseModel:
from hydra.utils import instantiate
from .config import instantiate

key = (id(self), name)
stack = _LAZY_LOADING_STACK.get()
Expand Down Expand Up @@ -1112,20 +1112,21 @@ def load_config(
self, cfg: "DictConfig", registry: ModelRegistry, skip_exceptions: bool = False, resolve_from: ModelRegistry | None = None
) -> ModelRegistry:
"""Load from OmegaConf DictConfig that follows hydra conventions."""
from .config import InstantiationException, instantiate

# Here we use hydra's 'instantiate' to instantiate models,
# because it provides a standard way to resolve the class name
# that's being constructed, through the "_target_" field.
# This also allows for nested attributes on the model itself to
# be constructed, even if they are not themselves of BaseModel type,
# or if they are of a specific subclass of the parent.
from hydra.errors import InstantiationException
from hydra.utils import instantiate
from omegaconf import OmegaConf, UnsupportedValueType

if resolve_from is not None and resolve_from is not registry:
initial_chain = [resolve_from, registry]
else:
initial_chain = [registry]

from omegaconf import OmegaConf, UnsupportedValueType

models_to_register = self._make_subregistries(cfg, initial_chain)
while True:
unresolved_models = []
Expand Down
67 changes: 67 additions & 0 deletions ccflow/config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
"""Expose the selected configuration backend through one API.

Hydra is preferred when available, otherwise Lerna is used. Set ``CCFLOW_CONFIG_BACKEND=lerna`` to require Lerna instead.
The backend is loaded on first API access, so importing ccflow does not require either package.
"""

import importlib
import os
from functools import cache
from types import ModuleType
from typing import TYPE_CHECKING

if TYPE_CHECKING:
from hydra import compose, initialize, initialize_config_dir, main
from hydra.core.hydra_config import HydraConfig
from hydra.errors import InstantiationException
from hydra.utils import instantiate

CONFIG_FRAMEWORK: str

__all__ = (
"CONFIG_FRAMEWORK",
"HydraConfig",
"InstantiationException",
"compose",
"initialize",
"initialize_config_dir",
"instantiate",
"main",
)

_ENV_VAR = "CCFLOW_CONFIG_BACKEND"
_SUBMODULES = {
"DefaultsList": "_internal.defaults_list",
"GlobalHydra": "core.global_hydra",
"HydraConfig": "core.hydra_config",
"InstantiationException": "errors",
"ObjectType": "core.object_type",
"RunMode": "types",
"instantiate": "utils",
}


@cache
def _get_framework() -> ModuleType:
requested = os.environ.get(_ENV_VAR, "").strip().lower()
frameworks = ("lerna",) if requested == "lerna" else ("hydra", "lerna")
for name in frameworks:
try:
return importlib.import_module(name)
except ModuleNotFoundError as error:
if error.name != name:
raise
if requested == "lerna":
raise ImportError(f"{_ENV_VAR}=lerna requires the 'lerna' package")
raise ImportError("Configuration requires 'hydra-core' or 'lerna'; install one to use ccflow.config")


def __getattr__(name: str):
if name == "CONFIG_FRAMEWORK":
return _get_framework().__name__
if name in ("main", "compose", "initialize", "initialize_config_dir"):
return getattr(_get_framework(), name)
if name in _SUBMODULES:
module = importlib.import_module(f"{_get_framework().__name__}.{_SUBMODULES[name]}")
return getattr(module, name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
5 changes: 2 additions & 3 deletions ccflow/examples/calculator/__main__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import hydra

from ccflow import config
from ccflow.utils.hydra import cfg_run

__all__ = ("main",)


@hydra.main(config_path="config", config_name="base", version_base=None)
@config.main(config_path="config", config_name="base", version_base=None)
def main(cfg):
cfg_run(cfg)

Expand Down
5 changes: 2 additions & 3 deletions ccflow/examples/etl/__main__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import hydra

from ccflow import config
from ccflow.utils.hydra import cfg_run

__all__ = ("main",)


@hydra.main(config_path="config", config_name="base", version_base=None)
@config.main(config_path="config", config_name="base", version_base=None)
def main(cfg):
cfg_run(cfg)

Expand Down
6 changes: 0 additions & 6 deletions ccflow/tests/config_user/sample2.yml

This file was deleted.

7 changes: 7 additions & 0 deletions ccflow/tests/enums/test_enums.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,18 @@ def auto():


class TestEnum(TestCase):
def setUp(self) -> None:
self._csp_modules = {name: module for name, module in sys.modules.items() if name == "csp" or name.startswith("csp.")}

def tearDown(self) -> None:
# Because test_init_parent and test_init_parent_csp muck around with imports
# Make sure we always rest the imports at the end of each test so that other
# tests are unaffected
os.environ.pop("CCFLOW_NO_CSP", None)
for name in list(sys.modules):
if name == "csp" or name.startswith("csp."):
sys.modules.pop(name)
sys.modules.update(self._csp_modules)
importlib.invalidate_caches()
import ccflow.enums

Expand Down
8 changes: 4 additions & 4 deletions ccflow/tests/examples/ai/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def _options(group: str) -> list[str]:


def _registry(*overrides):
return load_config(overrides=["checks=off", *overrides])
return load_config(overrides=["checks='off'", *overrides])


class RegistryTestCase(TestCase):
Expand Down Expand Up @@ -70,7 +70,7 @@ def test_every_group_offers_a_choice(self):
def test_every_option_composes_and_instantiates(self):
for group, option in ALL_OPTIONS:
with self.subTest(group=group, option=option):
self.assertIsNotNone(_registry(f"{group}={option}")["/task"])
self.assertIsNotNone(_registry(f"{group}={option!r}")["/task"])


class TestTasks(RegistryTestCase):
Expand Down Expand Up @@ -177,9 +177,9 @@ def test_checks_are_enforced_by_default(self):

def test_the_concise_agents_variant_inherits_every_session(self):
# `load_config` mutates the one root registry, so each value must be read before reloading.
default = load_config(overrides=["checks=off", "agents=poem"])["/task"].arbiter.session
default = load_config(overrides=["checks='off'", "agents=poem"])["/task"].arbiter.session
default_command, contract = default.command, default.output_contract
concise = load_config(overrides=["checks=off", "agents=poem_concise"])["/task"].arbiter.session
concise = load_config(overrides=["checks='off'", "agents=poem_concise"])["/task"].arbiter.session
self.assertNotEqual(concise.command, default_command)
self.assertEqual(concise.output_contract, contract)

Expand Down
43 changes: 38 additions & 5 deletions ccflow/tests/test_base_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@
from unittest import TestCase, mock

import pytest
from hydra.errors import InstantiationException
from omegaconf import OmegaConf
from omegaconf.errors import InterpolationKeyError
from pydantic import ConfigDict, Field

from ccflow import BaseModel, LazyRegistry, ModelRegistry, RegistryLookupContext, RootModelRegistry, model_alias
from ccflow.base import RegistryKeyError, resolve_str
from ccflow.config import InstantiationException


class MyTestModel(BaseModel):
Expand Down Expand Up @@ -461,6 +461,25 @@ def test_load_config(self):
r.load_config(cfg, overwrite=True)
self.assertEqual(r["foo"], m)

def test_load_config_uses_selected_framework(self):
from ccflow import config

cfg = OmegaConf.create(
{
"foo": {
"_target_": "ccflow.tests.test_base_registry.MyTestModel",
"a": "test",
"b": 0.0,
}
}
)
registry = ModelRegistry(name="test")

with mock.patch("ccflow.config.instantiate", wraps=config.instantiate) as instantiate:
registry.load_config(cfg)

instantiate.assert_called_once()

def test_load_config_with_function(self):
cfg = OmegaConf.create(
{
Expand Down Expand Up @@ -754,6 +773,17 @@ def test_defers_models_until_access(self):
self.assertIs(root["/lazy/group/source"], source)
self.assertEqual(LazyTestModel.constructions, 1)

def test_materialization_uses_selected_framework(self):
from ccflow import config

root = self._load_registry()

with mock.patch("ccflow.config.instantiate", wraps=config.instantiate) as instantiate:
source = root["/lazy/group/source"]

self.assertIsInstance(source, LazyTestModel)
instantiate.assert_called_once()

def test_materializes_dependency_closure(self):
root = self._load_registry()

Expand Down Expand Up @@ -870,7 +900,9 @@ def test_cross_registry_cycle_reports_full_path(self):
left["model"]

def test_concurrent_cross_registry_cycle_does_not_deadlock(self):
from hydra.utils import instantiate as hydra_instantiate
from ccflow import config

selected_instantiate = config.instantiate

root = ModelRegistry.root()
left = LazyRegistry(name="left")
Expand All @@ -890,10 +922,10 @@ def synchronized_instantiate(*args, **kwargs):
if not getattr(local, "started", False):
local.started = True
barrier.wait(timeout=2)
return hydra_instantiate(*args, **kwargs)
return selected_instantiate(*args, **kwargs)

with (
mock.patch("hydra.utils.instantiate", side_effect=synchronized_instantiate),
mock.patch("ccflow.config.instantiate", side_effect=synchronized_instantiate),
concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor,
):
futures = [executor.submit(registry.__getitem__, "model") for registry in (left, right)]
Expand Down Expand Up @@ -979,6 +1011,7 @@ def test_pending_config_is_read_only_copy(self):
self.assertEqual(registry["pending"].a, "pending")

def test_recursive_hydra_instantiation_is_rejected(self):
from hydra.errors import InstantiationException as HydraInstantiationException
from hydra.utils import instantiate

cfg = OmegaConf.create(
Expand All @@ -993,7 +1026,7 @@ def test_recursive_hydra_instantiation_is_rejected(self):
}
)

with self.assertRaisesRegex(InstantiationException, "Set '_recursive_: false'"):
with self.assertRaisesRegex(HydraInstantiationException, "Set '_recursive_: false'"):
instantiate(cfg, _convert_="all")

def test_resolve_from_applies_to_direct_pending_entry(self):
Expand Down
Loading
Loading