diff --git a/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md b/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md new file mode 100644 index 000000000..03b380130 --- /dev/null +++ b/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md @@ -0,0 +1,10 @@ +### Fixed + +- **`repo scan` no longer fails a path the repository's `.gitignore` ignores** — guidance + that names generated or ignored files had each such path reported `fail` as a rename, + because its parent directory exists and the file is never tracked. A missing path, directory + (with or without its trailing slash) or glob that a tracked `.gitignore` ignores is now + `unresolved`, and the reason names the `.gitignore` line. The rules follow gitignore(5), negation and nested files included; + `.git/info/exclude` and `core.excludesFile` are not part of the snapshot and are not read. + Some paths an earlier scan reported `fail` are now `unresolved`, so the fail count and the L0 + gate result can differ from a scan of the same repository made before this change. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 38d9c96d8..5a05bba8c 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1057,7 +1057,26 @@ A path in a code span is tried at the file's directory and each of its ancestors. If none has it, the path is `fail` when its parent directory exists at one of those bases (a rename or a deleted file), `ok_suffix` when it ends a longer path in the tree (a root file naming paths inside a subproject), and -`unresolved` otherwise (a build output, a path inside a submodule). `ok_suffix` +`unresolved` otherwise (a build output, a path inside a submodule). A missing +path that a tracked `.gitignore` ignores is `unresolved` rather than `fail`, +with the `.gitignore` line in the reason (also for a path that was `unresolved` +already): guidance that warns about generated files names paths no snapshot +holds. A directory counts as ignored when git ignores it, or, when no `!` line +names the directory itself (`!*/` names every directory, so it does not count), +would ignore every file probed directly inside it: a plain name and each name a +`!` line re-includes. So `include/*` ignores `include/`, and so does `out/*` with +`!*/`, while a whitelist of `*`, `!*/`, `!*.py` does not ignore a missing +`olddir/`, nor `!/setup.py` a missing `setup.py`. A path without a trailing slash counts when it +would be as a file or as a directory: under that whitelist `*` ignores a missing +`olddir` read as a file, so a directory needs its trailing slash to be checked +as a rename. A glob counts when one of the files it is +probed with would be: its wildcards filled in, and each name an ignoring line +matches that the glob matches too (`gen.h` for `pkg/g*.h`), when a literal +starts or ends the glob's last segment: `maintenance/*` or `pkg/*_*` would match +the name of any rule (`_skbuild`), which says nothing about the glob. The probe +tries a few names, not every name the glob matches. `.git/info/exclude` and +`core.excludesFile` belong to a clone, not the tree, +and are not read. `ok_suffix` and `ok_basename` resolve a path but cannot see it renamed, which then reads as `unresolved`; guidance the gate must protect names its paths from the repository root. A bare name diff --git a/dokimi/dokimi/repos/l0/core.py b/dokimi/dokimi/repos/l0/core.py index ba47e975d..043b841dc 100644 --- a/dokimi/dokimi/repos/l0/core.py +++ b/dokimi/dokimi/repos/l0/core.py @@ -21,6 +21,7 @@ from typing import Any from ..source import RepoTree, SourceError +from .gitignore import GitIgnore, Rule from .markdown import PYTHON_LANGS, SHELL_LANGS, STARLARK_LANGS, Fence, split # Files whose words are prose, not code: an identifier found only there is not in the code. @@ -85,6 +86,14 @@ def glob_regex(pat: str) -> re.Pattern[str]: return re.compile("".join(out) + r"\Z") +_FILLS = ("x", "dokimi-probe") # what a `*` becomes in a probe of the files a glob names + + +def _fill(pat: str, fill: str) -> str: + """``pat`` with its wildcards filled in: ``**/`` as nothing, ``*`` as ``fill``, ``?`` as its first letter.""" + return re.sub(r"\?", fill[0], re.sub(r"\*+", fill, pat.replace("**/", ""))) + + def is_doc(path: str) -> bool: name = posixpath.basename(path) if name == "CMakeLists.txt" or name.startswith("requirements"): @@ -305,11 +314,11 @@ def resolve_path(self, file: str, raw: str) -> tuple[str, str] | None: for c in cands: parent = posixpath.dirname(c) if parent and parent in self.dir_set: - return "fail", f"{parent}/ exists but {c} does not" + return self._missing([c], rel.endswith("/"), f"{parent}/ exists but {c} does not") hits = [x for x in self.by_name.get(posixpath.basename(target), []) if x.endswith("/" + target)] if hits: return "ok_suffix", f"below a directory the text does not name; {len(hits)} match(es), e.g. {hits[0]}" - return "unresolved", "tried " + ", ".join(cands) + return self._missing(cands, rel.endswith("/"), "tried " + ", ".join(cands), "unresolved") def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: """A glob is judged like the plain path it generalises (see :meth:`resolve_path`).""" @@ -327,13 +336,69 @@ def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: for b in bases: d = posixpath.normpath(posixpath.join(b, parent)) if d in self.dir_set: - return "fail", f"{d}/ exists but nothing in it matches {posixpath.basename(pat)}" + probes = self._probes(posixpath.normpath(posixpath.join(b, pat))) + return self._missing(probes, rel.endswith("/"), + f"{d}/ exists but nothing in it matches {posixpath.basename(pat)}") if parent: rx = glob_regex("*/" + pat) hit = next((x for x in every if rx.match(x)), None) if hit: return "ok_suffix", f"below a directory the text does not name; e.g. {hit}" - return "unresolved", f"glob {pat} matches nothing in the tree; a build may create it" + probes = [p for b in bases for p in self._probes(posixpath.normpath(posixpath.join(b, pat)))] + return self._missing(probes, rel.endswith("/"), + f"glob {pat} matches nothing in the tree; a build may create it", "unresolved") + + def _probes(self, full: str) -> list[str]: + """Files the glob ``full`` names, to ask the ``.gitignore`` about: a probe of a few names, not every one. + + Its wildcards filled in, and each name an ignoring rule matches that the glob matches too + (``gen.h`` for ``g*.h``); two fillings, so one re-included (``!x.cxx``) leaves the other. + Rule names count only for a last segment a literal starts or ends: ``*`` or ``*_*`` matches + the name of an unrelated rule (``_skbuild``), which is no evidence the glob is ignored. + """ + folder, name = posixpath.split(full) + rx = glob_regex(full) + named = name[:1] not in "*?[" or name[-1:] not in "*?]" + paths = [posixpath.join(_fill(folder, fill), n) for fill in _FILLS + for n in (_fill(name, fill), *(self.gitignore().names(fill) if named else ()))] + return [p for p in dict.fromkeys(paths) if rx.match(p)] + + def _missing(self, paths: list[str], is_dir: bool, why: str, status: str = "fail") -> tuple[str, str]: + """``status`` for a path the tree lacks, unless a ``.gitignore`` ignores it: a build output. + + ``paths`` are the files a glob names, or the path at each base it is tried from; any of them + being ignored is enough. + """ + rule = next(filter(None, (self._ignored_by(p, is_dir) for p in paths)), None) + if rule is None: + return status, why + return "unresolved", (f"{why}; {rule.source}:{rule.line} (`{rule.pattern}`) ignores it, " + "so it is likely generated rather than tracked") + + def _ignored_by(self, path: str, is_dir: bool) -> Rule | None: + """The rule that ignores ``path``, read as a directory too: without a trailing slash it may + still name one (``dist`` under ``dist/``). + + A path no ``!`` line names (``!/setup.py``, ``!*.py``; ``!*/`` names every directory, not this + one) also counts when every file probed directly inside it would be (``include/*``, or ``out/*`` + with ``!*/``): a plain name and each name a ``!`` line re-includes, so a whitelist (``*``, + ``!*/``, ``!*.py``) does not hide a renamed directory. + """ + ig = self.gitignore() + readings = (True,) if is_dir else (False, True) + rule = next(filter(None, (ig.ignored_by(path, d) for d in readings)), None) + if rule is not None: + return rule + if any(r and r.name.strip("*") for r in (ig.reincluded_by(path, d) for d in readings)): + return None + inside = [ig.ignored_by(posixpath.join(path, n)) for n in ("file", *ig.names("x", negated=True))] + return inside[0] if all(inside) else None + + def gitignore(self) -> GitIgnore: + def load() -> GitIgnore: + self.preload([f for f in self.files if posixpath.basename(f) == ".gitignore"]) + return GitIgnore(self.files, self.text) + return self.cached("gitignore", load) def _every(self) -> list[str]: return self.cached("every", lambda: sorted(self.file_set | self.dir_set)) diff --git a/dokimi/dokimi/repos/l0/gitignore.py b/dokimi/dokimi/repos/l0/gitignore.py new file mode 100644 index 000000000..09dcb0e55 --- /dev/null +++ b/dokimi/dokimi/repos/l0/gitignore.py @@ -0,0 +1,249 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Which paths a snapshot's ``.gitignore`` files ignore, read from the tree like every L0 fact. + +A missing path the repository ignores is a build output the guidance warns about, +not a rename: L0 reports it ``unresolved``. Only tracked ``.gitignore`` files count; +``.git/info/exclude`` and ``core.excludesFile`` are per clone and not part of the snapshot. + +The rules follow gitignore(5): ``#`` comments and blank lines skipped, ``!`` +re-includes, a trailing ``/`` matches directories only, a pattern with a ``/`` +before its end is anchored to its file's directory and one without matches a +name at any depth, ``*``, ``?`` and ``[...]`` stay within one segment, ``**`` spans +segments, a line git's wildmatch aborts on matches nothing, the last matching line wins with deeper files read after shallower +ones, and nothing under an ignored directory can be re-included. +""" + +from __future__ import annotations + +import posixpath +import re +from dataclasses import dataclass +from typing import Callable, Iterable + + +@dataclass(frozen=True) +class Rule: + source: str # the .gitignore path + line: int + pattern: str # the line as written + base: str # the directory the file is in, "" at the root + regex: re.Pattern[str] + anchored: bool + dir_only: bool + negated: bool + name: str = "" # the last segment of the pattern, wildcards kept; "" for a line git cannot read + + def matches(self, path: str, is_dir: bool) -> bool: + if self.dir_only and not is_dir: + return False + if self.base: + if not path.startswith(self.base + "/"): + return False + path = path[len(self.base) + 1:] + return bool(self.regex.match(path if self.anchored else posixpath.basename(path))) + + +def _segment(pat: str) -> str: + """Regex for a glob within path segments: ``*`` and ``?`` never cross ``/``. + + As in CPython's fnmatch, every ``*`` but the last takes the fixed-width run after it + in an atomic group, so a line with many stars cannot backtrack exponentially. + """ + runs: list[list[str]] = [[]] # the fixed-width atoms before, between and after the stars + i = 0 + while i < len(pat): + c = pat[i] + if c == "[": + atom, i = _bracket(pat, i) + runs[-1].append(atom) + continue + if c == "\\": + i += 1 + if i == len(pat): + raise _Invalid + runs[-1].append(re.escape(pat[i])) + elif c == "*": + if len(runs) == 1 or runs[-1]: + runs.append([]) + elif c == "?": + runs[-1].append("[^/]") + else: + runs[-1].append(re.escape(c)) + i += 1 + head, *rest = ("".join(r) for r in runs) + if not rest: + return head + *middle, tail = rest + return head + "".join(f"(?>[^/]*?{m})" for m in middle) + "[^/]*" + tail + + +class _Invalid(Exception): + """A line git's wildmatch aborts on (an unclosed ``[``, a trailing ``\\``): it matches nothing.""" + + +_CLASSES = { + "alnum": "a-zA-Z0-9", "alpha": "a-zA-Z", "blank": r" \t", "cntrl": r"\x00-\x1f\x7f", + "digit": "0-9", "graph": "!-~", "lower": "a-z", "print": " -~", "punct": r"!-/:-@\[-`{-~", + "space": r" \t\n\r", "upper": "A-Z", "xdigit": "0-9A-Fa-f", +} + + +def _bracket(pat: str, i: int) -> tuple[str, int]: + """The regex for the ``[...]`` at ``pat[i]`` and the index after it, read as git's wildmatch reads it. + + A ``]`` first is a member, ``\\`` escapes, ``[:name:]`` is an ASCII class, a reversed + range keeps only its start, and the set never matches ``/``. + """ + i += 1 + neg = pat[i:i + 1] in ("!", "^") + i += neg + items: list[str] = [] + prev = "" + while i < len(pat) and (pat[i] != "]" or not items): + c = pat[i] + if c == "\\" or (c == "-" and prev and pat[i + 1:i + 2] not in ("", "]")): + i += 1 + (c == "-" and pat[i + 1] == "\\") + if i == len(pat): + raise _Invalid + if c == "-": + items.append(f"{re.escape(prev)}-{re.escape(pat[i])}" if prev <= pat[i] else "") + prev = "" + else: + prev = pat[i] + items.append(re.escape(prev)) + elif c == "[" and pat[i + 1:i + 2] == ":" and (end := pat.find("]", i + 2)) > i + 2 and pat[end - 1] == ":": + if pat[i + 2:end - 1] not in _CLASSES: + raise _Invalid + items.append(_CLASSES[pat[i + 2:end - 1]]) + prev, i = "", end + else: + prev = c + items.append(re.escape(c)) + i += 1 + if i == len(pat): + raise _Invalid + return "(?!/)[" + ("^" if neg else "") + "".join(items) + "]", i + 1 + + +def _split(pat: str) -> list[str]: + """``pat`` cut at each ``/`` outside a bracket expression.""" + parts, start, i = [], 0, 0 + while i < len(pat): + if pat[i] == "[": + i = _bracket(pat, i)[1] + continue + if pat[i] == "/": + parts.append(pat[start:i]) + start = i + 1 + i += 1 + (pat[i] == "\\") + return parts + [pat[start:]] + + +def _regex(pat: str) -> str: + """``**/`` leading, ``/**/`` inside and ``/**`` trailing span segments; any other ``**`` is ``*``.""" + parts = _split(pat) + out: list[str] = [] + for i, part in enumerate(parts): + last = i == len(parts) - 1 + if part == "**": + out.append(".*" if last else "(?:[^/]+/)*") + continue + out.append(_segment(part) + ("" if last else "/")) + return "".join(out) + + +_NEVER = re.compile(r"(?!)") +_MEMBERS = "xad0_-.AZ9" # tried in turn for a ``[...]``: the first one it matches stands for it + + +def example(pat: str, fill: str) -> str | None: + """A name the one-segment ``pat`` matches: each ``*`` as ``fill``, each ``?`` as its first letter, + each ``[...]`` as a member; ``None`` when no candidate member fits a class.""" + out, i = [], 0 + while i < len(pat): + c = pat[i] + if c == "[": + atom, i = _bracket(pat, i) + out.append(next((m for m in fill[0] + _MEMBERS if re.fullmatch(atom, m)), "")) + if not out[-1]: + return None + continue + if c == "\\": + i += 1 + out.append(pat[i]) + else: + out.append({"*": fill, "?": fill[0]}.get(c, c)) + i += 1 + return "".join(out) + + +def _trim(line: str) -> str: + """``line`` without its trailing spaces, as git trims it: a tab stays, and so does a space after a ``\\``.""" + end, i = len(line), 0 + while i < len(line): + if line[i] != " ": + end = len(line) + i += line[i] == "\\" # the escaped character, a space or a second backslash + elif end == len(line): + end = i + i += 1 + return line[:end] + + +def parse(source: str, text: str) -> list[Rule]: + base = posixpath.dirname(source) + rules = [] + # git splits at "\n" only and drops a "\r" before it, and skips a byte-order mark + for n, raw in enumerate(text.removeprefix("\ufeff").split("\n"), 1): + line = _trim(raw.removesuffix("\r")) + if not line or line.startswith("#"): + continue + negated = line.startswith("!") + pat = line[1:] if negated else line + if pat.startswith(("\\#", "\\!")): + pat = pat[1:] + dir_only = pat.endswith("/") + pat = pat.rstrip("/") + if not pat: + continue + anchored = "/" in pat + try: + regex, name = re.compile(_regex(pat.lstrip("/")) + r"\Z"), _split(pat.lstrip("/"))[-1] + except _Invalid: + regex, name = _NEVER, "" + rules.append(Rule(source, n, line, base, regex, anchored, dir_only, negated, name)) + return rules + + +class GitIgnore: + """The ignore rules of every ``.gitignore`` in a tree.""" + + def __init__(self, files: Iterable[str], read: Callable[[str], str]) -> None: + sources = sorted((f for f in files if posixpath.basename(f) == ".gitignore"), + key=lambda f: (f.count("/"), f)) + self.rules = [r for f in sources for r in parse(f, read(f))] + + def _last(self, path: str, is_dir: bool) -> Rule | None: + return next((r for r in reversed(self.rules) if r.matches(path, is_dir)), None) + + def names(self, fill: str, negated: bool = False) -> list[str]: + """A name the last segment of each ignoring rule matches (``gen_a.h`` for ``gen_[a-c].h``), + or with ``negated`` of each ``!`` rule.""" + found = (example(r.name, fill) for r in self.rules if r.name and r.negated == negated) + return list(dict.fromkeys(n for n in found if n)) + + def reincluded_by(self, path: str, is_dir: bool = False) -> Rule | None: + """The ``!`` line that matches ``path`` itself last, or ``None`` when the last match ignores it or none does.""" + rule = self._last(posixpath.normpath(path).strip("/"), is_dir) + return rule if rule is not None and rule.negated else None + + def ignored_by(self, path: str, is_dir: bool = False) -> Rule | None: + """The rule that ignores ``path`` or one of its directories, or ``None`` when git would not.""" + parts = posixpath.normpath(path).strip("/").split("/") + for i in range(1, len(parts) + 1): + rule = self._last("/".join(parts[:i]), is_dir or i < len(parts)) + if rule is not None and not rule.negated: + return rule + return None diff --git a/dokimi/tests/test_agent_profiles.py b/dokimi/tests/test_agent_profiles.py index 48e728e98..8d26cf12c 100644 --- a/dokimi/tests/test_agent_profiles.py +++ b/dokimi/tests/test_agent_profiles.py @@ -442,10 +442,11 @@ def test_the_api_name_check_would_actually_fire(): contaminated = ( "Call asnumpy() once at the edge, pass usm_type='device' and sycl_queue=q to " - "dpctl.tensor.empty(), and set dtype=float32." + "dpnp.tensor.empty(), and set dtype=float32." ) - leaked = {m for shape in API_SHAPES for m in re.findall(shape, contaminated)} - assert leaked, "the API-name guard is vacuous" + # One name per shape: a dotted call, a bare call, a keyword argument. Each needs its own. + for shape, name in zip(API_SHAPES, ("tensor.empty(", "asnumpy()", "usm_type='"), strict=True): + assert name in re.findall(shape, contaminated), f"{shape} no longer catches {name}" def test_a_persona_that_paraphrases_its_skill_is_detected(): diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index c8763ffcf..c11f0fbea 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -84,6 +84,57 @@ def _wf(matrix: str) -> dict[str, str]: {("path", "third/dep/lib.h"): "unresolved"}), ("outside the repository", "`../../x.md`", {}, {("path", "../../x.md"): "fail"}), ("bare directory name", "`metrics/`", {"pkg/metrics/a.py": ""}, {("path", "metrics/"): "ok_basename"}), + # a missing path the repository's .gitignore ignores is ignored + ("gitignored file", "`pkg/gen.h`", {"pkg/.gitignore": "*.h\n", "pkg/a.c": ""}, + {("path", "pkg/gen.h"): "unresolved"}), + ("gitignored directory contents", "`pkg/include/`", {"pkg/.gitignore": "include/*\n", "pkg/a.c": ""}, + {("path", "pkg/include/"): "unresolved"}), + ("gitignored glob", "`pkg/*.cxx`", {"pkg/.gitignore": "*.cxx\n", "pkg/a.pyx": ""}, + {("path", "pkg/*.cxx"): "unresolved"}), + ("gitignored glob in a nested guidance file", "x", + {"pkg/AGENTS.md": "`sub/*.cxx`\n", "pkg/sub/.gitignore": "*.cxx\n", "pkg/sub/a.pyx": ""}, + {("path", "sub/*.cxx"): "unresolved"}), + ("gitignored directory written without a slash", "`pkg/dist`", + {"pkg/.gitignore": "dist/\n", "pkg/a.c": ""}, {("path", "pkg/dist"): "unresolved"}), + ("a whitelisting .gitignore keeps a missing file a rename", "`pkg/old.c`", + {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.c\n", "pkg/a.c": ""}, {("path", "pkg/old.c"): "fail"}), + ("an anchored re-include under a whitelist keeps a missing path a rename", "`pkg/setup.py` `pkg/olddir/`", + {"pkg/.gitignore": "*\n!*/\n!/.gitignore\n!/setup.py\n!/olddir/\n", "pkg/keep.txt": ""}, + {("path", "pkg/setup.py"): "fail", ("path", "pkg/olddir/"): "fail"}), + ("gitignored glob, one name it could stand for re-included", "`pkg/*.cxx`", + {"pkg/.gitignore": "*.cxx\n!x.cxx\n", "pkg/a.pyx": ""}, {("path", "pkg/*.cxx"): "unresolved"}), + ("gitignored directory contents, no trailing slash", "`pkg/include`", + {"pkg/.gitignore": "include/*\n", "pkg/a.c": ""}, {("path", "pkg/include"): "unresolved"}), + ("a whitelisting .gitignore keeps a missing directory a rename", "`pkg/olddir/`", + {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.py\n", "pkg/a.py": ""}, {("path", "pkg/olddir/"): "fail"}), + ("a whitelisting .gitignore keeps a missing directory glob a rename", "`pkg/old*/`", + {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.py\n", "pkg/a.py": ""}, {("path", "pkg/old*/"): "fail"}), + ("under a whitelist a path without a slash is ignored as a file", "`pkg/Makefile` `pkg/olddir`", + {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.py\n", "pkg/a.py": ""}, + {("path", "pkg/Makefile"): "unresolved", ("path", "pkg/olddir"): "unresolved"}), + ("a directory re-included by `!*/`, its files still ignored", "`pkg/out/` `pkg/tmp/`", + {"pkg/.gitignore": "out/*\ntmp/**\n!*/\n", "pkg/a.c": ""}, + {("path", "pkg/out/"): "unresolved", ("path", "pkg/tmp/"): "unresolved"}), + ("a directory with one file re-included is a rename", "`pkg/include/` `pkg/include`", + {"pkg/.gitignore": "include/*\n!include/keep.h\n", "pkg/a.c": ""}, + {("path", "pkg/include/"): "fail", ("path", "pkg/include"): "fail"}), + ("gitignored glob the fill-in name misses", "`img/*.png` `pkg/g*.h` `pkg/gen_*.h` `pkg/gen*`", + {"img/.gitignore": "[0-9]*.png\n", "img/a.c": "", "pkg/.gitignore": "gen.h\ngen_[a-c].h\n", "pkg/a.c": ""}, + {("path", "img/*.png"): "unresolved", ("path", "pkg/g*.h"): "unresolved", + ("path", "pkg/gen_*.h"): "unresolved", ("path", "pkg/gen*"): "unresolved"}), + ("a glob of wildcards and no literal end is not probed with rule names", "`pkg/*` `pkg/*_*`", + {"pkg/.gitignore": "_skbuild\n", "pkg/a.py": ""}, {("path", "pkg/*"): "ok", ("path", "pkg/*_*"): "fail"}), + ("a glob no ignoring rule's names fit", "`pkg/*.c`", {"pkg/.gitignore": "gen.h\n", "pkg/a.h": ""}, + {("path", "pkg/*.c"): "fail"}), + ("gitignored directory glob", "`pkg/gen*/`", {"pkg/.gitignore": "gen*/\n", "pkg/a.c": ""}, + {("path", "pkg/gen*/"): "unresolved"}), + ("re-included by the .gitignore: a rename again", "`pkg/keep.h`", + {"pkg/.gitignore": "*.h\n!keep.h\n", "pkg/a.c": ""}, {("path", "pkg/keep.h"): "fail"}), + ("ignored elsewhere only", "`pkg/gen.h`", {"other/.gitignore": "*.h\n", "pkg/a.c": ""}, + {("path", "pkg/gen.h"): "fail"}), + ("a .gitignore line Python's re cannot compile", "`pkg/z.h` `pkg/old.c`", + {"pkg/.gitignore": "[z-a].h\n", "pkg/a.c": ""}, + {("path", "pkg/z.h"): "unresolved", ("path", "pkg/old.c"): "fail"}), # links ("link outside the repository", "[x](../../x.md)", {}, {("link", "../../x.md"): "fail"}), ("percent-encoded link", "[x](docs/my%20file.md)", {"docs/my file.md": ""}, @@ -519,6 +570,32 @@ def test_a_path_into_a_submodule_says_so(tmp_path): assert "inside the submodule third/dep" in payload["claims"][0]["why"] +def test_a_glob_an_unrelated_rule_spells_names_no_rule(tmp_path): + files = {".gitignore": "_skbuild\n", "pkg/a.py": ""} + claim = _scan(tmp_path, "`maintenance/*`\n", files)["claims"][0] + assert (claim["status"], claim["why"]) == ("unresolved", "glob maintenance/* matches nothing in the tree; " + "a build may create it") + + +@pytest.mark.parametrize("agents, files, line", [ + ("`pkg/include/`", {"pkg/.gitignore": "*.o\ninclude/*\n", "pkg/a.c": ""}, "pkg/.gitignore:2 (`include/*`)"), + # the parent directory is missing too + ("`pkg/resources/cmake/`", {".gitignore": "pkg/resources/cmake\n", "pkg/a.py": ""}, + ".gitignore:1 (`pkg/resources/cmake`)"), + ("`pkg/include/pkg/*.h`", {".gitignore": "pkg/include/pkg/*.h\n", "pkg/a.py": ""}, + ".gitignore:1 (`pkg/include/pkg/*.h`)"), + # tried from the root first; the guidance file's own directory is the one ignored + ("x", {"pkg/AGENTS.md": "`gen/out.h`\n", "pkg/.gitignore": "/gen/\n", "pkg/a.c": ""}, + "pkg/.gitignore:1 (`/gen/`)"), + ("x", {"pkg/AGENTS.md": "`resources/*.h`\n", "pkg/.gitignore": "/resources/*.h\n", "pkg/a.c": ""}, + "pkg/.gitignore:1 (`/resources/*.h`)"), +]) +def test_a_gitignored_path_names_the_line_that_ignores_it(tmp_path, agents, files, line): + claim = next(c for c in _scan(tmp_path, agents + "\n", files)["claims"] if c["kind"] == "path") + assert claim["status"] == "unresolved" + assert f"{line} ignores it" in claim["why"], claim["why"] + + def test_a_missing_pyargs_module_names_the_nearest_package(tmp_path): payload = _scan(tmp_path, _shell("pytest --pyargs pkg.tests.gone pkg.tests.sub.test_b lib.tests.test_a " "other.tests.test_c"), diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py new file mode 100644 index 000000000..019c9e9b6 --- /dev/null +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -0,0 +1,185 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The L0 ``.gitignore`` matcher agrees with ``git check-ignore`` on the same files.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from dokimi.repos.l0.gitignore import GitIgnore + +IGNORES = { + ".gitignore": "\n".join([ + "# a comment, then a blank line", + "", + "*.o", + "!keep.o", + "build/", + "!build/keep.txt", + "/top.txt", + "docs/**/gen", + "**/cache", + "logs/**", + "a/**/b", + "data/*.csv", + r"\#hash", + "[abc].tmp", + "[!x]y.bin", + "include/*", + "[z-a].h", # a reversed range keeps only its start + "[[:digit:]].d", + r"[\]]x.e", + "[a/b]x", # a bracket's / does not split the pattern + "d[!a]e/f", # nor does a bracket match one + "[]a]q", + "a[", # unclosed: matches nothing + r"\!bang", + "sp\\ ", # an escaped trailing space is kept + "[^x]z.bin", + "[a-]w", + "[[:digit:]]9.d", + "tab\t", # git trims spaces, not tabs + "bs\\\\ ", # an escaped backslash does not keep the space after it + "", + ]), + "sub/.gitignore": "!*.o\nlocal\n", + "crlf/.gitignore": "\ufeffbom\r\nlone\rcr\n", # git skips the mark, splits at \n only, drops \r before it +} + +PATHS = [ + "x.o", "deep/x.o", "keep.o", "deep/keep.o", "sub/x.o", "sub/deeper/x.o", + "build/a/b.c", "src/build/x.c", "build/keep.txt", "notbuild/x.c", + "top.txt", "sub/top.txt", + "docs/gen", "docs/a/b/gen", "other/docs/gen", + "cache", "x/cache/y.txt", + "logs/a/b.txt", "logs.txt", + "a/b", "a/x/y/b", "a/bb", + "data/x.csv", "data/sub/x.csv", + "#hash", "hash", + "a.tmp", "d.tmp", + "ay.bin", "xy.bin", + "include/x.h", "sub/include/x.h", + "z.h", "y.h", "1.d", "a.d", "]x.e", "ax", "bx", "sub/ax", "a[", + "d/e/f", "dbe/f", "]q", "aq", + "sub/local", "local", "sub/deeper/local", + "!bang", "sp ", "sp", "yz.bin", "xz.bin", "-w", "aw", "99.d", + "tab\t", "tab", "bs\\", "bs\\ ", "crlf/bom", "crlf/lone\rcr", "crlf/lone", "crlf/cr", +] + + +# What `git check-ignore` printed for PATHS, so the matcher is checked on a host without git too. +GIT_IGNORED = { + "x.o", "deep/x.o", "build/a/b.c", "src/build/x.c", "build/keep.txt", "top.txt", + "docs/gen", "docs/a/b/gen", "cache", "x/cache/y.txt", "logs/a/b.txt", "a/b", "a/x/y/b", + "data/x.csv", "#hash", "a.tmp", "ay.bin", "include/x.h", "z.h", "1.d", "]x.e", "ax", "bx", + "dbe/f", "]q", "aq", "sub/local", "sub/deeper/local", "!bang", "sp ", "yz.bin", "-w", "aw", + "99.d", "tab\t", "bs\\", "crlf/bom", "crlf/lone\rcr", +} + + +def _git_ignored(root: Path) -> set[str]: + # HOME and XDG_CONFIG_HOME too: the default core.excludesFile is $XDG_CONFIG_HOME/git/ignore, + # and git before 2.32 reads $HOME/.gitconfig whatever GIT_CONFIG_GLOBAL says + env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", + "HOME": str(root), "XDG_CONFIG_HOME": str(root)} + # No template: a host init.templateDir could bring info/exclude rules + subprocess.run(["git", "init", "-q", "--template=", str(root)], check=True, env=env) + for rel, text in IGNORES.items(): + (root / rel).parent.mkdir(parents=True, exist_ok=True) + (root / rel).write_text(text, encoding="utf-8") + # --no-index matches the names alone, so PATHS (a tab, a \r, a trailing space) need no files + out = subprocess.run(["git", "-C", str(root), "check-ignore", "--no-index", "--stdin", "-z"], + input="\0".join(PATHS).encode(), capture_output=True, env=env, check=False) + assert out.returncode in (0, 1), out.stderr # 1: nothing ignored + return set(out.stdout.decode().split("\0")) - {""} # -z and bytes: names unquoted, a \r kept + + +def test_the_matcher_ignores_what_git_ignores(): + rules = GitIgnore(IGNORES, IGNORES.__getitem__) + assert {p for p in PATHS if rules.ignored_by(p) is not None} == GIT_IGNORED + + +# Windows git may read a \ in a path as a separator, and init sets core.ignorecase there +@pytest.mark.skipif(shutil.which("git") is None or sys.platform == "win32", + reason="needs a POSIX git") +def test_git_still_ignores_what_the_matcher_is_checked_against(tmp_path): + assert _git_ignored(tmp_path) == GIT_IGNORED + + +@pytest.mark.parametrize("path, source, line", [ + ("deep/x.o", ".gitignore", 3), + # Nothing under an ignored directory comes back, so the directory's line is the reason. + ("build/keep.txt", ".gitignore", 5), + ("sub/deeper/local", "sub/.gitignore", 2), +]) +def test_the_rule_that_ignores_a_path_is_named(path, source, line): + rule = GitIgnore(IGNORES, IGNORES.__getitem__).ignored_by(path) + assert rule is not None and (rule.source, rule.line) == (source, line) + + +def test_a_directory_only_rule_needs_a_directory(): + rules = GitIgnore({".gitignore": "out/\n"}, lambda _: "out/\n") + assert rules.ignored_by("out") is None + assert rules.ignored_by("out", is_dir=True) is not None + + +@pytest.mark.parametrize("line, path", [("x\\", "x\\"), ("[[:foo:]]x", "f]x"), ("[a-\\", "[a-\\")]) +def test_a_line_git_cannot_read_matches_nothing(line, path): + assert GitIgnore({".gitignore": line}, lambda _: line).ignored_by(path) is None + + +# git check-ignore ignores nothing for each of these lines: none leaves a pattern +@pytest.mark.parametrize("line", ["", " ", "/", "//", "!", "!/"]) +def test_a_line_with_no_pattern_is_no_rule(line): + text = f"gen.h\n{line}\n" + rules = GitIgnore({".gitignore": text}, lambda _: text) + assert [r.line for r in rules.rules] == [1] + assert rules.ignored_by("a/b") is None and rules.ignored_by("a", is_dir=True) is None + assert rules.reincluded_by("gen.h") is None and rules.names("x", negated=True) == [] + + +def test_a_tree_without_a_gitignore_ignores_nothing(): + rules = GitIgnore(["src/a.py", "README.md"], lambda f: pytest.fail(f"read {f}")) + assert rules.ignored_by("src/a.py") is None and rules.names("x") == [] + + +def test_many_stars_do_not_backtrack(): + # A backtracking [^/]* per star takes ~15 s here; the line and the path are repository content. + line = "*a" * 9 + "*b" + rules = GitIgnore({".gitignore": line}, lambda _: line) + start = time.perf_counter() + assert rules.ignored_by("pkg/" + "a" * 45) is None + assert rules.ignored_by("pkg/" + "a" * 45 + "b") is not None + assert time.perf_counter() - start < 1 + + +@pytest.mark.parametrize("line, name, path", [ + ("gen.h", "gen.h", "pkg/gen.h"), ("[0-9]*.png", "0x.png", "img/0x.png"), + ("gen_[a-c].h", "gen_a.h", "gen_a.h"), ("[!x]?.o", "ax.o", "ax.o"), ("[[:upper:]]*", "Ax", "Ax"), + ("\\*.h", "*.h", "*.h"), ("build/out*/", "outx", "build/outx/f"), + ("[_]gen.h", "_gen.h", "_gen.h"), ("[-]gen.h", "-gen.h", "-gen.h"), ("[.]gen.h", ".gen.h", ".gen.h"), +]) +def test_a_rule_names_a_file_it_ignores(line, name, path): + rules = GitIgnore({".gitignore": line}, lambda _: line) + assert rules.names("x") == [name] + assert rules.ignored_by(path) is not None + + +def test_a_class_no_candidate_fits_names_nothing(): + line = "[!a-zA-Z0-9_.-]*.o\ngen.h\n" + assert GitIgnore({".gitignore": line}, lambda _: line).names("x") == ["gen.h"] + + +def test_a_negation_is_named_only_when_asked_for(): + line = "gen.h\n!keep.o\n" + rules = GitIgnore({".gitignore": line}, lambda _: line) + assert rules.names("x") == ["gen.h"] + assert rules.names("x", negated=True) == ["keep.o"]