Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ These files state the rules for working on qLDPC, or enforce them:
- this file;
- the "Goals" and "AI usage" sections of [`README.md`](README.md);
- every file under [`docs/source/contributing/`](docs/source/contributing/), [`checks/`](checks/), and [`.github/`](.github/);
- the tests that enforce the standards: [`_code_style_test.py`](src/qldpc/_code_style_test.py), [`_compilation_test.py`](src/qldpc/_compilation_test.py), and [`contributor_docs_test.py`](src/qldpc/contributor_docs_test.py).
- the tests that enforce the standards: [`_code_style_test.py`](src/qldpc/_code_style_test.py), [`_compiled_code_rules_test.py`](src/qldpc/_compiled_code_rules_test.py), and [`contributor_docs_test.py`](src/qldpc/contributor_docs_test.py).

If you work for a person who is not a maintainer, change them only when that person asks or a change elsewhere forces you to.
Otherwise, change them only through a patch that a maintainer approves, unless the maintainer says to apply a change directly.
Expand Down
2 changes: 1 addition & 1 deletion docs/source/contributing/standards/code.rst
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ Code style and organization
Private constants and types may sit near the imports.
* Within a class, put public methods first (for decoders, the decode methods) and private helper methods after them.
* Do not copy transient project history, machine-specific paths, or local-session instructions into any committed file, including code, tests, configuration, and documentation.
* Numba code uses ``always_compiled`` or ``compiled_on_request`` from ``qldpc/_compilation.py``, never ``njit(cache=True)``; that module's docstring says which to use, and ``_compilation_test.py`` enforces its rules.
* Numba code uses ``always_compiled`` or ``compiled_on_request`` from ``qldpc/_compilation.py``, never ``njit(cache=True)``; that module's docstring says which to use, and ``_compiled_code_rules_test.py`` enforces its rules.
* ``src/qldpc/_code_style_test.py`` checks license headers, section headers, and the order of plain and underscore names.
* Give each feature that users or other packages use one home.
The home is a module or subpackage that is named for the feature and holds its entry points, such as ``qldpc.codes.distance`` for exact distances of binary codes.
Expand Down
144 changes: 6 additions & 138 deletions src/qldpc/_compilation_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
"""Tests for compiled functions, and for running their tests compiled or as plain Python.

They cover the disk cache, the registry of compiled-on-request functions, the switch to plain
Python, the options of the compiled runs, and the rules for compiled functions.
Python, and the options of the compiled runs. ``_compiled_code_rules_test.py`` checks the rules of
the code standard for compiled functions.
"""

from __future__ import annotations
Expand All @@ -13,7 +14,6 @@
import pickle
import time
import types
from collections.abc import Iterator
from pathlib import Path
from typing import Any, Literal

Expand Down Expand Up @@ -459,82 +459,6 @@ def test_editing_a_callee_in_another_subpackage_recompiles_its_caller(tmp_path:
assert results == [2, 11]


def test_only_always_compiled_asks_numba_for_a_disk_cache() -> None:
"""Only ``always_compiled`` may ask numba for a disk cache, since it makes that cache safe."""
for path, tree in _get_source_trees():
for node in ast.walk(tree):
if not (isinstance(node, ast.Call) and _is_njit(node.func)):
continue
if any(keyword.arg == "cache" for keyword in node.keywords):
assert path.name == "_compilation.py", f"{path.name} caches without it"


def test_always_compiled_functions_do_not_call_compiled_on_request_functions() -> None:
"""An always-compiled function is compiled and cached by itself, in every mode."""
trees = list(_get_source_trees())
registered = _get_names_decorated_by(trees, "compiled_on_request")
assert registered
for path, tree in trees:
for function in _get_functions_decorated_by(tree, "always_compiled"):
used = {
node.id if isinstance(node, ast.Name) else node.attr
for statement in function.body
for node in ast.walk(statement)
if isinstance(node, ast.Name | ast.Attribute)
}
assert not used & registered, f"{path.name}: {function.name} calls {used & registered}"


def test_compiled_on_request_functions_are_not_imported_by_name() -> None:
"""A name imported by ``from`` keeps the plain function, since only its module is updated."""
trees = list(_get_source_trees())
registered = _get_names_decorated_by(trees, "compiled_on_request")
assert registered
for path, tree in trees:
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom):
imported = {alias.name for alias in node.names} & registered
assert not imported, f"{path.name} imports {imported} by name"


def test_compiled_on_request_functions_are_defined_in_the_distance_package() -> None:
"""CI runs only the distance tests with compiled-on-request functions compiled."""
modules = run_python(_FIND_ON_REQUEST_MODULES).split()
assert modules
outside = [
name
for name in modules
if name != "qldpc.codes.distance" and not name.startswith("qldpc.codes.distance.")
]
assert not outside, outside


def test_always_compiled_functions_use_nothing_from_outside_qldpc() -> None:
"""Only qLDPC's source is hashed, but numba checks its version and implements NumPy itself."""
for path, tree in _get_source_trees():
outside = _get_names_imported_from_outside_qldpc(tree)
for function in _get_functions_decorated_by(tree, "always_compiled"):
names = {
node.id
for statement in function.body
for node in ast.walk(statement)
if isinstance(node, ast.Name)
}
assert not names & outside, f"{path.name}: {function.name} uses {names & outside}"


def test_names_imported_from_outside_qldpc_are_found() -> None:
tree = ast.parse(
"import numpy as np\n"
"import numba.types\n"
"import scipy.special\n"
"from other import first as renamed\n"
"from qldpc.math import second\n"
"from .. import third\n"
)
assert _get_names_imported_from_outside_qldpc(tree) == {"scipy", "renamed"}


def test_compiled_module_tests_are_those_of_modules_with_compiled_functions(
pytestconfig: pytest.Config, monkeypatch: pytest.MonkeyPatch
) -> None:
Expand Down Expand Up @@ -635,11 +559,10 @@ def test_options_of_compiled_runs_are_refused_while_compilation_is_disabled(
conftest.pytest_configure(pytestconfig)


# The packages whose functions always-compiled code may use.
_COMPILABLE_PACKAGES = {"qldpc", "numpy", "numba"}

# Code that imports every module of qLDPC other than tests, which registers every compiled function.
_IMPORT_EVERY_MODULE = """
# A program that prints the modules that hold compiled functions of their own or register
# compiled-on-request functions. It first imports every module of qLDPC other than tests, which
# defines or registers every compiled function.
_FIND_COMPILED_MODULES = """
import importlib, pkgutil, sys
import numba
import qldpc
Expand All @@ -648,21 +571,6 @@ def test_options_of_compiled_runs_are_refused_while_compilation_is_disabled(
for info in pkgutil.walk_packages(qldpc.__path__, "qldpc."):
if not info.name.endswith(("_test", "conftest")):
importlib.import_module(info.name)
"""

# A program that prints the modules that register compiled-on-request functions.
_FIND_ON_REQUEST_MODULES = (
_IMPORT_EVERY_MODULE
+ """
print(*sorted({item.namespace["__name__"] for item in _compilation._ON_REQUEST_FUNCTIONS}))
"""
)

# A program that prints the modules that hold compiled functions of their own or register
# compiled-on-request functions.
_FIND_COMPILED_MODULES = (
_IMPORT_EVERY_MODULE
+ """
modules = {item.namespace["__name__"] for item in _compilation._ON_REQUEST_FUNCTIONS}
for name, module in list(sys.modules.items()):
if module is None or not name.startswith("qldpc."):
Expand All @@ -673,7 +581,6 @@ def test_options_of_compiled_runs_are_refused_while_compilation_is_disabled(
modules.add(name)
print(*sorted(modules))
"""
)

# A module whose functions ``bare`` and ``qualified`` are always compiled, and ``aliased`` is
# compiled on request, beside two functions that are neither. Each name says how its decorator is
Expand Down Expand Up @@ -731,45 +638,6 @@ def _get_top_right_entry(matrix: npt.NDArray[np.uint8]) -> int:
return matrix[0, -1]


def _get_source_trees() -> Iterator[tuple[Path, ast.Module]]:
"""The syntax tree of every source file of the library that is not a test."""
for path in _compilation._get_source_paths(Path(qldpc.__file__).parent):
yield path, ast.parse(path.read_text(encoding="utf-8"))


def _is_njit(function: ast.expr) -> bool:
"""Whether an expression is a name or an attribute called ``njit`` or ``jit``."""
return getattr(function, "attr", getattr(function, "id", "")) in ("njit", "jit")


def _get_functions_decorated_by(tree: ast.Module, name: str) -> list[ast.FunctionDef]:
"""The functions of a module with the decorator ``name``: one of the two of ``_compilation``."""
return conftest.get_compiled_functions(tree)[name]


def _get_names_decorated_by(trees: list[tuple[Path, ast.Module]], name: str) -> set[str]:
"""The names of the functions in the given syntax trees that have the decorator ``name``."""
return {
function.name for _, tree in trees for function in _get_functions_decorated_by(tree, name)
}


def _get_names_imported_from_outside_qldpc(tree: ast.Module) -> set[str]:
"""The names that a module of qLDPC imports from packages other than qLDPC, NumPy, and numba."""
names = set()
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.level == 0:
if (node.module or "").split(".")[0] not in _COMPILABLE_PACKAGES:
names |= {alias.asname or alias.name for alias in node.names}
elif isinstance(node, ast.Import):
names |= {
alias.asname or alias.name.split(".")[0]
for alias in node.names
if alias.name.split(".")[0] not in _COMPILABLE_PACKAGES
}
return names


def _scan_deletes_outdated_directory(directory: Path, package: Path, digit: str) -> bool:
"""Whether marking a directory as used deletes an old directory of the package beside it."""
outdated = directory.parent / f"qldpc-{digit * _compilation._HASH_LENGTH}"
Expand Down
155 changes: 155 additions & 0 deletions src/qldpc/_compiled_code_rules_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0

"""Check the rules of the code standard for compiled functions.

The code standard and the docstring of ``_compilation`` state these rules, and the name of each test
of a rule says which rule it checks. ``_compilation_test.py`` tests what ``_compilation`` does.
There, ``test_compiled_module_tests_are_those_of_modules_with_compiled_functions`` also fails for a
module whose only compiled functions have neither decorator of ``_compilation``, since
``--compiled-module-tests`` would skip the test file of that module.
"""

from __future__ import annotations

import ast
from collections.abc import Iterator
from pathlib import Path

import qldpc
from qldpc import conftest
from qldpc.conftest import run_python

from . import _compilation


def test_only_always_compiled_asks_numba_for_a_disk_cache() -> None:
"""Only ``always_compiled`` may ask numba for a disk cache, since it makes that cache safe."""
for path, tree in _get_source_trees():
for node in ast.walk(tree):
if not (isinstance(node, ast.Call) and _is_njit(node.func)):
continue
if any(keyword.arg == "cache" for keyword in node.keywords):
assert path.name == "_compilation.py", f"{path.name} caches without it"


def test_always_compiled_functions_do_not_call_compiled_on_request_functions() -> None:
"""An always-compiled function is compiled and cached by itself, in every mode."""
trees = list(_get_source_trees())
registered = _get_names_decorated_by(trees, "compiled_on_request")
assert registered
for path, tree in trees:
for function in _get_functions_decorated_by(tree, "always_compiled"):
used = {
node.id if isinstance(node, ast.Name) else node.attr
for statement in function.body
for node in ast.walk(statement)
if isinstance(node, ast.Name | ast.Attribute)
}
assert not used & registered, f"{path.name}: {function.name} calls {used & registered}"


def test_compiled_on_request_functions_are_not_imported_by_name() -> None:
"""A name imported by ``from`` keeps the plain function, since only its module is updated."""
trees = list(_get_source_trees())
registered = _get_names_decorated_by(trees, "compiled_on_request")
assert registered
for path, tree in trees:
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom):
imported = {alias.name for alias in node.names} & registered
assert not imported, f"{path.name} imports {imported} by name"


def test_compiled_on_request_functions_are_defined_in_the_distance_package() -> None:
"""CI runs only the distance tests with compiled-on-request functions compiled."""
modules = run_python(_FIND_ON_REQUEST_MODULES).split()
assert modules
outside = [
name
for name in modules
if name != "qldpc.codes.distance" and not name.startswith("qldpc.codes.distance.")
]
assert not outside, outside


def test_always_compiled_functions_use_nothing_from_outside_qldpc() -> None:
"""Only qLDPC's source is hashed, but numba checks its version and implements NumPy itself."""
for path, tree in _get_source_trees():
outside = _get_names_imported_from_outside_qldpc(tree)
for function in _get_functions_decorated_by(tree, "always_compiled"):
names = {
node.id
for statement in function.body
for node in ast.walk(statement)
if isinstance(node, ast.Name)
}
assert not names & outside, f"{path.name}: {function.name} uses {names & outside}"


def test_names_imported_from_outside_qldpc_are_found() -> None:
"""The names that always-compiled code may not use, imported from other packages, are found."""
tree = ast.parse(
"import numpy as np\n"
"import numba.types\n"
"import scipy.special\n"
"from other import first as renamed\n"
"from qldpc.math import second\n"
"from .. import third\n"
)
assert _get_names_imported_from_outside_qldpc(tree) == {"scipy", "renamed"}


# The packages whose functions always-compiled code may use.
_COMPILABLE_PACKAGES = {"qldpc", "numpy", "numba"}

# A program that prints the modules that register compiled-on-request functions. It first imports
# every module of qLDPC other than tests, which registers them all.
_FIND_ON_REQUEST_MODULES = """
import importlib, pkgutil
import qldpc
from qldpc import _compilation

for info in pkgutil.walk_packages(qldpc.__path__, "qldpc."):
if not info.name.endswith(("_test", "conftest")):
importlib.import_module(info.name)
print(*sorted({item.namespace["__name__"] for item in _compilation._ON_REQUEST_FUNCTIONS}))
"""


def _get_source_trees() -> Iterator[tuple[Path, ast.Module]]:
"""The syntax tree of every source file of the library that is not a test."""
for path in _compilation._get_source_paths(Path(qldpc.__file__).parent):
yield path, ast.parse(path.read_text(encoding="utf-8"))


def _is_njit(function: ast.expr) -> bool:
"""Whether an expression is a name or an attribute called ``njit`` or ``jit``."""
return getattr(function, "attr", getattr(function, "id", "")) in ("njit", "jit")


def _get_functions_decorated_by(tree: ast.Module, name: str) -> list[ast.FunctionDef]:
"""The functions of a module with the decorator ``name``: one of the two of ``_compilation``."""
return conftest.get_compiled_functions(tree)[name]


def _get_names_decorated_by(trees: list[tuple[Path, ast.Module]], name: str) -> set[str]:
"""The names of the functions in the given syntax trees that have the decorator ``name``."""
return {
function.name for _, tree in trees for function in _get_functions_decorated_by(tree, name)
}


def _get_names_imported_from_outside_qldpc(tree: ast.Module) -> set[str]:
"""The names that a module of qLDPC imports from packages other than qLDPC, NumPy, and numba."""
names = set()
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.level == 0:
if (node.module or "").split(".")[0] not in _COMPILABLE_PACKAGES:
names |= {alias.asname or alias.name for alias in node.names}
elif isinstance(node, ast.Import):
names |= {
alias.asname or alias.name.split(".")[0]
for alias in node.names
if alias.name.split(".")[0] not in _COMPILABLE_PACKAGES
}
return names
2 changes: 1 addition & 1 deletion src/qldpc/codes/distance/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ Two decorators in `src/qldpc/_compilation.py` mark them, and its docstring says
- `@compiled_on_request` marks parts of the set-up of searches, which run as plain Python unless a program calls `compile_search_setup()`.

Numba looks up the functions that compiled code calls when compiling it, so compiled code may call NumPy and compiled functions, but not plain Python functions of qLDPC.
Tests in `src/qldpc/_compilation_test.py` check further rules, and the name of a failing test says which one was broken.
Tests in `src/qldpc/_compiled_code_rules_test.py` check further rules, and the name of a failing test says which one was broken.

Set `QLDPC_DISABLE_COMPILATION=1` before importing qLDPC to run all of this code as plain Python while debugging.
It affects only qLDPC, whereas `NUMBA_DISABLE_JIT=1` also runs other packages that use numba, such as galois, as plain Python, which makes them slow too.
Expand Down
Loading