From 69635f13a318367e4b97dc14c529e23757c92ba2 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Mon, 26 Oct 2026 14:20:31 -0700 Subject: [PATCH 01/14] fix(scan): a missing path the repository's .gitignore ignores is unresolved (not failed) Guidance that warns about ignored files names paths the tree never Signed-off-by: Nikita Grigorian --- ...50000_ndgrigorian_scan-gitignored-paths.md | 8 ++ dokimi/docs/repo-evaluation-contract.md | 9 +- dokimi/dokimi/repos/l0/core.py | 23 +++- dokimi/dokimi/repos/l0/gitignore.py | 123 ++++++++++++++++++ dokimi/tests/test_repo_l0_cases.py | 16 +++ dokimi/tests/test_repo_l0_gitignore.py | 92 +++++++++++++ 6 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md create mode 100644 dokimi/dokimi/repos/l0/gitignore.py create mode 100644 dokimi/tests/test_repo_l0_gitignore.py diff --git a/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md b/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md new file mode 100644 index 000000000..5e34bce75 --- /dev/null +++ b/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md @@ -0,0 +1,8 @@ +### Fixed + +- **`repo scan` no longer fails a path the repository's `.gitignore` ignores** — guidance + that names generated or ignored files had each such path reported `fail` as a rename, + because its parent directory exists and the file is never tracked. A missing path, directory + or glob that a tracked `.gitignore` ignores is now `unresolved`, and the reason names the + `.gitignore` line. The rules follow gitignore(5), negation and nested files included; + `.git/info/exclude` and `core.excludesFile` are not part of the snapshot and are not read. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e9f75e1a0..84f58dcaf 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -888,7 +888,14 @@ A path in a code span is tried at the file's directory and each of its ancestors. If none has it, the path is `fail` when its parent directory exists at one of those bases (a rename or a deleted file), `ok_suffix` when it ends a longer path in the tree (a root file naming paths inside a subproject), and -`unresolved` otherwise (a build output, a path inside a submodule). `ok_suffix` +`unresolved` otherwise (a build output, a path inside a submodule). A missing +path that a tracked `.gitignore` ignores is `unresolved` rather than `fail`, +with the `.gitignore` line in the reason: guidance that warns about generated +files names paths no snapshot holds. +A directory counts as ignored when a file directly inside it would be, +and a glob when one file it names would be; +`.git/info/exclude` and `core.excludesFile` belong to a clone, not the tree, +and are not read. `ok_suffix` and `ok_basename` resolve a path but cannot see it renamed, which then reads as `unresolved`; guidance the gate must protect names its paths from the repository root. A bare name diff --git a/dokimi/dokimi/repos/l0/core.py b/dokimi/dokimi/repos/l0/core.py index 08460a2c9..b4e9899fa 100644 --- a/dokimi/dokimi/repos/l0/core.py +++ b/dokimi/dokimi/repos/l0/core.py @@ -21,6 +21,7 @@ from typing import Any from ..source import RepoTree, SourceError +from .gitignore import GitIgnore from .markdown import PYTHON_LANGS, SHELL_LANGS, STARLARK_LANGS, Fence, split # Files whose words are prose, not code: an identifier found only there is not in the code. @@ -305,7 +306,7 @@ def resolve_path(self, file: str, raw: str) -> tuple[str, str] | None: for c in cands: parent = posixpath.dirname(c) if parent and parent in self.dir_set: - return "fail", f"{parent}/ exists but {c} does not" + return self._missing(c, rel.endswith("/"), f"{parent}/ exists but {c} does not") hits = [x for x in self.by_name.get(posixpath.basename(target), []) if x.endswith("/" + target)] if hits: return "ok_suffix", f"below a directory the text does not name; {len(hits)} match(es), e.g. {hits[0]}" @@ -327,7 +328,9 @@ def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: for b in bases: d = posixpath.normpath(posixpath.join(b, parent)) if d in self.dir_set: - return "fail", f"{d}/ exists but nothing in it matches {posixpath.basename(pat)}" + instance = posixpath.join(b, re.sub(r"\*+|\?", "x", pat.replace("**/", ""))) + return self._missing(instance, False, + f"{d}/ exists but nothing in it matches {posixpath.basename(pat)}") if parent: rx = glob_regex("*/" + pat) hit = next((x for x in every if rx.match(x)), None) @@ -335,6 +338,22 @@ def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: return "ok_suffix", f"below a directory the text does not name; e.g. {hit}" return "unresolved", f"glob {pat} matches nothing in the tree; a build may create it" + def _missing(self, path: str, is_dir: bool, why: str) -> tuple[str, str]: + """``fail`` for a path the tree lacks, unless a ``.gitignore`` ignores it: a build output. + + A directory counts as ignored when a file directly inside it would be (``include/*``). + """ + rule = self.gitignore().ignored_by(posixpath.join(path, "file") if is_dir else path) + if rule is None: + return "fail", why + return "unresolved", f"{why}; {rule.source}:{rule.line} (`{rule.pattern}`) ignores it, so it is not meant to be tracked" + + def gitignore(self) -> GitIgnore: + def load() -> GitIgnore: + self.preload([f for f in self.files if posixpath.basename(f) == ".gitignore"]) + return GitIgnore(self.files, self.text) + return self.cached("gitignore", load) + def _every(self) -> list[str]: return self.cached("every", lambda: sorted(self.file_set | self.dir_set)) diff --git a/dokimi/dokimi/repos/l0/gitignore.py b/dokimi/dokimi/repos/l0/gitignore.py new file mode 100644 index 000000000..047d6acd9 --- /dev/null +++ b/dokimi/dokimi/repos/l0/gitignore.py @@ -0,0 +1,123 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Which paths a snapshot's ``.gitignore`` files ignore, read from the tree like every L0 fact. + +A missing path the repository ignores is a build output the guidance warns about, +not a rename: L0 reports it ``unresolved``. Only tracked ``.gitignore`` files count; +``.git/info/exclude`` and ``core.excludesFile`` are per clone and not part of the snapshot. + +The rules follow gitignore(5): ``#`` comments and blank lines skipped, ``!`` +re-includes, a trailing ``/`` matches directories only, a pattern with a ``/`` +before its end is anchored to its file's directory and one without matches a +name at any depth, ``*`` and ``?`` stay within one segment, ``**`` spans +segments, the last matching line wins with deeper files read after shallower +ones, and nothing under an ignored directory can be re-included. +""" + +from __future__ import annotations + +import posixpath +import re +from dataclasses import dataclass +from typing import Callable, Iterable + + +@dataclass(frozen=True) +class Rule: + source: str # the .gitignore path + line: int + pattern: str # the line as written + base: str # the directory the file is in, "" at the root + regex: re.Pattern[str] + anchored: bool + dir_only: bool + negated: bool + + def matches(self, path: str, is_dir: bool) -> bool: + if self.dir_only and not is_dir: + return False + if self.base: + if not path.startswith(self.base + "/"): + return False + path = path[len(self.base) + 1:] + return bool(self.regex.match(path if self.anchored else posixpath.basename(path))) + + +def _segment(pat: str) -> str: + """Regex for a glob within path segments: ``*`` and ``?`` never cross ``/``.""" + out, i = [], 0 + while i < len(pat): + c = pat[i] + if c == "\\" and i + 1 < len(pat): + out.append(re.escape(pat[i + 1])) + i += 2 + continue + if c == "*": + out.append("[^/]*") + elif c == "?": + out.append("[^/]") + elif c == "[" and (end := pat.find("]", i + 2)) != -1: + body = pat[i + 1:end] + neg = body[:1] in ("!", "^") + out.append("[" + ("^" if neg else "") + body[neg:].replace("\\", "\\\\") + "]") + i = end + else: + out.append(re.escape(c)) + i += 1 + return "".join(out) + + +def _regex(pat: str) -> str: + """``**/`` leading, ``/**/`` inside and ``/**`` trailing span segments; any other ``**`` is ``*``.""" + parts = pat.split("/") + out: list[str] = [] + for i, part in enumerate(parts): + last = i == len(parts) - 1 + if part == "**": + out.append(".*" if last else "(?:[^/]+/)*") + continue + out.append(_segment(part) + ("" if last else "/")) + return "".join(out) + + +def parse(source: str, text: str) -> list[Rule]: + base = posixpath.dirname(source) + rules = [] + for n, raw in enumerate(text.splitlines(), 1): + line = re.sub(r"(? None: + sources = sorted((f for f in files if posixpath.basename(f) == ".gitignore"), + key=lambda f: (f.count("/"), f)) + self.rules = [r for f in sources for r in parse(f, read(f))] + + def _last(self, path: str, is_dir: bool) -> Rule | None: + return next((r for r in reversed(self.rules) if r.matches(path, is_dir)), None) + + def ignored_by(self, path: str, is_dir: bool = False) -> Rule | None: + """The rule that ignores ``path`` or one of its directories, or ``None`` when git would not.""" + parts = posixpath.normpath(path).strip("/").split("/") + for i in range(1, len(parts) + 1): + rule = self._last("/".join(parts[:i]), is_dir or i < len(parts)) + if rule is not None and not rule.negated: + return rule + return None diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index d92f38785..00b2804b2 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -71,6 +71,17 @@ def _shell(*lines: str) -> str: {("path", "third/dep/lib.h"): "unresolved"}), ("outside the repository", "`../../x.md`", {}, {("path", "../../x.md"): "fail"}), ("bare directory name", "`metrics/`", {"pkg/metrics/a.py": ""}, {("path", "metrics/"): "ok_basename"}), + # a missing path the repository's .gitignore ignores is ignored + ("gitignored file", "`pkg/gen.h`", {"pkg/.gitignore": "*.h\n", "pkg/a.c": ""}, + {("path", "pkg/gen.h"): "unresolved"}), + ("gitignored directory contents", "`pkg/include/`", {"pkg/.gitignore": "include/*\n", "pkg/a.c": ""}, + {("path", "pkg/include/"): "unresolved"}), + ("gitignored glob", "`pkg/*.cxx`", {"pkg/.gitignore": "*.cxx\n", "pkg/a.pyx": ""}, + {("path", "pkg/*.cxx"): "unresolved"}), + ("re-included by the .gitignore: a rename again", "`pkg/keep.h`", + {"pkg/.gitignore": "*.h\n!keep.h\n", "pkg/a.c": ""}, {("path", "pkg/keep.h"): "fail"}), + ("ignored elsewhere only", "`pkg/gen.h`", {"other/.gitignore": "*.h\n", "pkg/a.c": ""}, + {("path", "pkg/gen.h"): "fail"}), # links ("link outside the repository", "[x](../../x.md)", {}, {("link", "../../x.md"): "fail"}), ("percent-encoded link", "[x](docs/my%20file.md)", {"docs/my file.md": ""}, @@ -279,6 +290,11 @@ def test_a_path_into_a_submodule_says_so(tmp_path): assert "inside the submodule third/dep" in payload["claims"][0]["why"] +def test_a_gitignored_path_names_the_line_that_ignores_it(tmp_path): + payload = _scan(tmp_path, "`pkg/include/`\n", {"pkg/.gitignore": "*.o\ninclude/*\n", "pkg/a.c": ""}) + assert "pkg/.gitignore:2 (`include/*`) ignores it" in payload["claims"][0]["why"] + + def test_no_claim_for_options_or_operators(tmp_path): got = _verdicts(_scan(tmp_path, _shell("make -j 8 all > log && pytest -k slow_case tests/"), {"Makefile": "all:\n", "tests/a.py": ""})) diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py new file mode 100644 index 000000000..a46c1be70 --- /dev/null +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -0,0 +1,92 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The L0 ``.gitignore`` matcher agrees with ``git check-ignore`` on the same files.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +from dokimi.repos.l0.gitignore import GitIgnore + +IGNORES = { + ".gitignore": "\n".join([ + "# a comment, then a blank line", + "", + "*.o", + "!keep.o", + "build/", + "!build/keep.txt", + "/top.txt", + "docs/**/gen", + "**/cache", + "logs/**", + "a/**/b", + "data/*.csv", + r"\#hash", + "[abc].tmp", + "[!x]y.bin", + "include/*", + "", + ]), + "sub/.gitignore": "!*.o\nlocal\n", +} + +PATHS = [ + "x.o", "deep/x.o", "keep.o", "deep/keep.o", "sub/x.o", "sub/deeper/x.o", + "build/a/b.c", "src/build/x.c", "build/keep.txt", "notbuild/x.c", + "top.txt", "sub/top.txt", + "docs/gen", "docs/a/b/gen", "other/docs/gen", + "cache", "x/cache/y.txt", + "logs/a/b.txt", "logs.txt", + "a/b", "a/x/y/b", "a/bb", + "data/x.csv", "data/sub/x.csv", + "#hash", "hash", + "a.tmp", "d.tmp", + "ay.bin", "xy.bin", + "include/x.h", "sub/include/x.h", + "sub/local", "local", "sub/deeper/local", +] + + +def _git_ignored(root: Path) -> set[str]: + env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"} + out = subprocess.run(["git", "-C", str(root), "check-ignore", "--no-index", "--", *PATHS], + capture_output=True, text=True, env=env, check=False) + assert out.returncode in (0, 1), out.stderr # 1: nothing ignored + return set(out.stdout.split("\n")) - {""} + + +def test_the_matcher_ignores_what_git_ignores(tmp_path): + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + for rel, text in IGNORES.items(): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / rel).write_text(text, encoding="utf-8") + for rel in PATHS: + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / rel).write_text("", encoding="utf-8") + + rules = GitIgnore(IGNORES, IGNORES.__getitem__) + ours = {p for p in PATHS if rules.ignored_by(p) is not None} + assert ours == _git_ignored(tmp_path) + + +@pytest.mark.parametrize("path, source, line", [ + ("deep/x.o", ".gitignore", 3), + # Nothing under an ignored directory comes back, so the directory's line is the reason. + ("build/keep.txt", ".gitignore", 5), + ("sub/deeper/local", "sub/.gitignore", 2), +]) +def test_the_rule_that_ignores_a_path_is_named(path, source, line): + rule = GitIgnore(IGNORES, IGNORES.__getitem__).ignored_by(path) + assert rule is not None and (rule.source, rule.line) == (source, line) + + +def test_a_directory_only_rule_needs_a_directory(): + rules = GitIgnore({".gitignore": "out/\n"}, lambda _: "out/\n") + assert rules.ignored_by("out") is None + assert rules.ignored_by("out", is_dir=True) is not None From 9163ea3ff7aca7b67d1cacfbeed7d3f8915bc902 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Mon, 26 Oct 2026 15:56:48 -0700 Subject: [PATCH 02/14] Fix outdated typo dpctl.tensor was removed previously, moved to dpnp Signed-off-by: Nikita Grigorian --- dokimi/tests/test_agent_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/tests/test_agent_profiles.py b/dokimi/tests/test_agent_profiles.py index 48e728e98..ed9863df2 100644 --- a/dokimi/tests/test_agent_profiles.py +++ b/dokimi/tests/test_agent_profiles.py @@ -442,7 +442,7 @@ def test_the_api_name_check_would_actually_fire(): contaminated = ( "Call asnumpy() once at the edge, pass usm_type='device' and sycl_queue=q to " - "dpctl.tensor.empty(), and set dtype=float32." + "dpnp.tensor.empty(), and set dtype=float32." ) leaked = {m for shape in API_SHAPES for m in re.findall(shape, contaminated)} assert leaked, "the API-name guard is vacuous" From 38e1a82cfb0c02cac075db9037ba7ffffaf2de05 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 27 Oct 2026 10:53:00 +0000 Subject: [PATCH 03/14] Show the rule as git read it in the gitignore reason Signed-off-by: Anton Volkov --- dokimi/dokimi/repos/l0/gitignore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/l0/gitignore.py b/dokimi/dokimi/repos/l0/gitignore.py index 047d6acd9..02d09feea 100644 --- a/dokimi/dokimi/repos/l0/gitignore.py +++ b/dokimi/dokimi/repos/l0/gitignore.py @@ -97,7 +97,7 @@ def parse(source: str, text: str) -> list[Rule]: if not pat: continue anchored = "/" in pat - rules.append(Rule(source, n, raw.strip(), base, re.compile(_regex(pat.lstrip("/")) + r"\Z"), + rules.append(Rule(source, n, line, base, re.compile(_regex(pat.lstrip("/")) + r"\Z"), anchored, dir_only, negated)) return rules From ad212fb7d79280103497f7eb49c95cb5d4ccfe1c Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 27 Oct 2026 02:54:47 -0700 Subject: [PATCH 04/14] fix(repo): read .gitignore brackets as git's wildmatch does A bracket body was copied into a Python character class, so a tracked line such as `[z-a].h` raised re.error and ended repo scan. Brackets now follow wildmatch (POSIX classes, escapes, `]` first, a `/` inside), and a line wildmatch aborts on (unclosed `[`, trailing `\`) matches nothing. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/repos/l0/gitignore.py | 96 ++++++++++++++++++++++---- dokimi/tests/test_repo_l0_cases.py | 3 + dokimi/tests/test_repo_l0_gitignore.py | 14 ++++ 3 files changed, 99 insertions(+), 14 deletions(-) diff --git a/dokimi/dokimi/repos/l0/gitignore.py b/dokimi/dokimi/repos/l0/gitignore.py index 02d09feea..b1bab0187 100644 --- a/dokimi/dokimi/repos/l0/gitignore.py +++ b/dokimi/dokimi/repos/l0/gitignore.py @@ -10,8 +10,8 @@ The rules follow gitignore(5): ``#`` comments and blank lines skipped, ``!`` re-includes, a trailing ``/`` matches directories only, a pattern with a ``/`` before its end is anchored to its file's directory and one without matches a -name at any depth, ``*`` and ``?`` stay within one segment, ``**`` spans -segments, the last matching line wins with deeper files read after shallower +name at any depth, ``*``, ``?`` and ``[...]`` stay within one segment, ``**`` spans +segments, a line git's wildmatch aborts on matches nothing, the last matching line wins with deeper files read after shallower ones, and nothing under an ignored directory can be re-included. """ @@ -49,28 +49,90 @@ def _segment(pat: str) -> str: out, i = [], 0 while i < len(pat): c = pat[i] - if c == "\\" and i + 1 < len(pat): - out.append(re.escape(pat[i + 1])) - i += 2 + if c == "[": + atom, i = _bracket(pat, i) + out.append(atom) continue - if c == "*": + if c == "\\": + i += 1 + if i == len(pat): + raise _Invalid + out.append(re.escape(pat[i])) + elif c == "*": out.append("[^/]*") elif c == "?": out.append("[^/]") - elif c == "[" and (end := pat.find("]", i + 2)) != -1: - body = pat[i + 1:end] - neg = body[:1] in ("!", "^") - out.append("[" + ("^" if neg else "") + body[neg:].replace("\\", "\\\\") + "]") - i = end else: out.append(re.escape(c)) i += 1 return "".join(out) +class _Invalid(Exception): + """A line git's wildmatch aborts on (an unclosed ``[``, a trailing ``\\``): it matches nothing.""" + + +_CLASSES = { + "alnum": "a-zA-Z0-9", "alpha": "a-zA-Z", "blank": r" \t", "cntrl": r"\x00-\x1f\x7f", + "digit": "0-9", "graph": "!-~", "lower": "a-z", "print": " -~", "punct": r"!-/:-@\[-`{-~", + "space": r" \t\n\r", "upper": "A-Z", "xdigit": "0-9A-Fa-f", +} + + +def _bracket(pat: str, i: int) -> tuple[str, int]: + """The regex for the ``[...]`` at ``pat[i]`` and the index after it, read as git's wildmatch reads it. + + A ``]`` first is a member, ``\\`` escapes, ``[:name:]`` is an ASCII class, a reversed + range keeps only its start, and the set never matches ``/``. + """ + i += 1 + neg = pat[i:i + 1] in ("!", "^") + i += neg + items: list[str] = [] + prev = "" + while i < len(pat) and (pat[i] != "]" or not items): + c = pat[i] + if c == "\\" or (c == "-" and prev and pat[i + 1:i + 2] not in ("", "]")): + i += 1 + (c == "-" and pat[i + 1] == "\\") + if i == len(pat): + raise _Invalid + if c == "-": + items.append(f"{re.escape(prev)}-{re.escape(pat[i])}" if prev <= pat[i] else "") + prev = "" + else: + prev = pat[i] + items.append(re.escape(prev)) + elif c == "[" and pat[i + 1:i + 2] == ":" and (end := pat.find("]", i + 2)) > i + 2 and pat[end - 1] == ":": + if pat[i + 2:end - 1] not in _CLASSES: + raise _Invalid + items.append(_CLASSES[pat[i + 2:end - 1]]) + prev, i = "", end + else: + prev = c + items.append(re.escape(c)) + i += 1 + if i == len(pat): + raise _Invalid + return "(?!/)[" + ("^" if neg else "") + "".join(items) + "]", i + 1 + + +def _split(pat: str) -> list[str]: + """``pat`` cut at each ``/`` outside a bracket expression.""" + parts, start, i = [], 0, 0 + while i < len(pat): + if pat[i] == "[": + i = _bracket(pat, i)[1] + continue + if pat[i] == "/": + parts.append(pat[start:i]) + start = i + 1 + i += 1 + (pat[i] == "\\") + return parts + [pat[start:]] + + def _regex(pat: str) -> str: """``**/`` leading, ``/**/`` inside and ``/**`` trailing span segments; any other ``**`` is ``*``.""" - parts = pat.split("/") + parts = _split(pat) out: list[str] = [] for i, part in enumerate(parts): last = i == len(parts) - 1 @@ -81,6 +143,9 @@ def _regex(pat: str) -> str: return "".join(out) +_NEVER = re.compile(r"(?!)") + + def parse(source: str, text: str) -> list[Rule]: base = posixpath.dirname(source) rules = [] @@ -97,8 +162,11 @@ def parse(source: str, text: str) -> list[Rule]: if not pat: continue anchored = "/" in pat - rules.append(Rule(source, n, line, base, re.compile(_regex(pat.lstrip("/")) + r"\Z"), - anchored, dir_only, negated)) + try: + regex = re.compile(_regex(pat.lstrip("/")) + r"\Z") + except _Invalid: + regex = _NEVER + rules.append(Rule(source, n, line, base, regex, anchored, dir_only, negated)) return rules diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index 00b2804b2..90be07e94 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -82,6 +82,9 @@ def _shell(*lines: str) -> str: {"pkg/.gitignore": "*.h\n!keep.h\n", "pkg/a.c": ""}, {("path", "pkg/keep.h"): "fail"}), ("ignored elsewhere only", "`pkg/gen.h`", {"other/.gitignore": "*.h\n", "pkg/a.c": ""}, {("path", "pkg/gen.h"): "fail"}), + ("a .gitignore line Python's re cannot compile", "`pkg/z.h` `pkg/old.c`", + {"pkg/.gitignore": "[z-a].h\n", "pkg/a.c": ""}, + {("path", "pkg/z.h"): "unresolved", ("path", "pkg/old.c"): "fail"}), # links ("link outside the repository", "[x](../../x.md)", {}, {("link", "../../x.md"): "fail"}), ("percent-encoded link", "[x](docs/my%20file.md)", {"docs/my file.md": ""}, diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py index a46c1be70..5f7bbdaf3 100644 --- a/dokimi/tests/test_repo_l0_gitignore.py +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -31,6 +31,13 @@ "[abc].tmp", "[!x]y.bin", "include/*", + "[z-a].h", # a reversed range keeps only its start + "[[:digit:]].d", + r"[\]]x.e", + "[a/b]x", # a bracket's / does not split the pattern + "d[!a]e/f", # nor does a bracket match one + "[]a]q", + "a[", # unclosed: matches nothing "", ]), "sub/.gitignore": "!*.o\nlocal\n", @@ -49,6 +56,8 @@ "a.tmp", "d.tmp", "ay.bin", "xy.bin", "include/x.h", "sub/include/x.h", + "z.h", "y.h", "1.d", "a.d", "]x.e", "ax", "bx", "sub/ax", "a[", + "d/e/f", "dbe/f", "]q", "aq", "sub/local", "local", "sub/deeper/local", ] @@ -90,3 +99,8 @@ def test_a_directory_only_rule_needs_a_directory(): rules = GitIgnore({".gitignore": "out/\n"}, lambda _: "out/\n") assert rules.ignored_by("out") is None assert rules.ignored_by("out", is_dir=True) is not None + + +@pytest.mark.parametrize("line, path", [("x\\", "x\\"), ("[[:foo:]]x", "f]x"), ("[a-\\", "[a-\\")]) +def test_a_line_git_cannot_read_matches_nothing(line, path): + assert GitIgnore({".gitignore": line}, lambda _: line).ignored_by(path) is None From 793fa29ce584c570638874832944a332ffe42944 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 27 Oct 2026 02:56:20 -0700 Subject: [PATCH 05/14] fix(repo): keep many-star .gitignore lines from backtracking Each `*` was an unbounded `[^/]*`, so a line like `*a*a*a*a*a*a*a*a*a*a*b` against a long missing name made repo scan hang. As in CPython's fnmatch, every star but the last now takes the fixed run after it atomically. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/repos/l0/gitignore.py | 26 ++++++++++++++++++-------- dokimi/tests/test_repo_l0_gitignore.py | 11 +++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/repos/l0/gitignore.py b/dokimi/dokimi/repos/l0/gitignore.py index b1bab0187..372671bfd 100644 --- a/dokimi/dokimi/repos/l0/gitignore.py +++ b/dokimi/dokimi/repos/l0/gitignore.py @@ -45,27 +45,37 @@ def matches(self, path: str, is_dir: bool) -> bool: def _segment(pat: str) -> str: - """Regex for a glob within path segments: ``*`` and ``?`` never cross ``/``.""" - out, i = [], 0 + """Regex for a glob within path segments: ``*`` and ``?`` never cross ``/``. + + As in CPython's fnmatch, every ``*`` but the last takes the fixed-width run after it + in an atomic group, so a line with many stars cannot backtrack exponentially. + """ + runs: list[list[str]] = [[]] # the fixed-width atoms before, between and after the stars + i = 0 while i < len(pat): c = pat[i] if c == "[": atom, i = _bracket(pat, i) - out.append(atom) + runs[-1].append(atom) continue if c == "\\": i += 1 if i == len(pat): raise _Invalid - out.append(re.escape(pat[i])) + runs[-1].append(re.escape(pat[i])) elif c == "*": - out.append("[^/]*") + if len(runs) == 1 or runs[-1]: + runs.append([]) elif c == "?": - out.append("[^/]") + runs[-1].append("[^/]") else: - out.append(re.escape(c)) + runs[-1].append(re.escape(c)) i += 1 - return "".join(out) + head, *rest = ("".join(r) for r in runs) + if not rest: + return head + *middle, tail = rest + return head + "".join(f"(?>[^/]*?{m})" for m in middle) + "[^/]*" + tail class _Invalid(Exception): diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py index 5f7bbdaf3..1b08881ef 100644 --- a/dokimi/tests/test_repo_l0_gitignore.py +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -7,6 +7,7 @@ import os import subprocess +import time from pathlib import Path import pytest @@ -104,3 +105,13 @@ def test_a_directory_only_rule_needs_a_directory(): @pytest.mark.parametrize("line, path", [("x\\", "x\\"), ("[[:foo:]]x", "f]x"), ("[a-\\", "[a-\\")]) def test_a_line_git_cannot_read_matches_nothing(line, path): assert GitIgnore({".gitignore": line}, lambda _: line).ignored_by(path) is None + + +def test_many_stars_do_not_backtrack(): + # A backtracking [^/]* per star takes ~15 s here; the line and the path are repository content. + line = "*a" * 9 + "*b" + rules = GitIgnore({".gitignore": line}, lambda _: line) + start = time.perf_counter() + assert rules.ignored_by("pkg/" + "a" * 45) is None + assert rules.ignored_by("pkg/" + "a" * 45 + "b") is not None + assert time.perf_counter() - start < 1 From 3e67c56ad38775f5c56f2880d14745ce40d63be7 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 27 Oct 2026 04:43:41 -0700 Subject: [PATCH 06/14] test(repo): pin the base of a gitignored glob in a nested guidance file Every gitignore case scanned a root AGENTS.md, so dropping the guidance file's directory from the glob probe in _glob left all tests green while a nested `sub/*.cxx` turned from unresolved to fail. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_repo_l0_cases.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index 90be07e94..aea925463 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -78,6 +78,9 @@ def _shell(*lines: str) -> str: {("path", "pkg/include/"): "unresolved"}), ("gitignored glob", "`pkg/*.cxx`", {"pkg/.gitignore": "*.cxx\n", "pkg/a.pyx": ""}, {("path", "pkg/*.cxx"): "unresolved"}), + ("gitignored glob in a nested guidance file", "x", + {"pkg/AGENTS.md": "`sub/*.cxx`\n", "pkg/sub/.gitignore": "*.cxx\n", "pkg/sub/a.pyx": ""}, + {("path", "sub/*.cxx"): "unresolved"}), ("re-included by the .gitignore: a rename again", "`pkg/keep.h`", {"pkg/.gitignore": "*.h\n!keep.h\n", "pkg/a.c": ""}, {("path", "pkg/keep.h"): "fail"}), ("ignored elsewhere only", "`pkg/gen.h`", {"other/.gitignore": "*.h\n", "pkg/a.c": ""}, From 62e8eb682262e231d150de257394fbca294262bf Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 27 Oct 2026 13:54:41 +0000 Subject: [PATCH 07/14] test(repo): pin gitignore line cleaning and class forms against git Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_l0_gitignore.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py index 1b08881ef..3473951d7 100644 --- a/dokimi/tests/test_repo_l0_gitignore.py +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -39,6 +39,11 @@ "d[!a]e/f", # nor does a bracket match one "[]a]q", "a[", # unclosed: matches nothing + r"\!bang", + "sp\\ ", # an escaped trailing space is kept + "[^x]z.bin", + "[a-]w", + "[[:digit:]]9.d", "", ]), "sub/.gitignore": "!*.o\nlocal\n", @@ -60,6 +65,7 @@ "z.h", "y.h", "1.d", "a.d", "]x.e", "ax", "bx", "sub/ax", "a[", "d/e/f", "dbe/f", "]q", "aq", "sub/local", "local", "sub/deeper/local", + "!bang", "sp ", "sp", "yz.bin", "xz.bin", "-w", "aw", "99.d", ] From 2f780a70b1deb37c021ba86246e90b50687d3bc5 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 27 Oct 2026 13:37:08 -0700 Subject: [PATCH 08/14] Address review comments * Fix x probe to be more general * Fix for missing directories when lacking trailing forward slash Signed-off-by: Nikita Grigorian --- ...50000_ndgrigorian_scan-gitignored-paths.md | 4 +-- dokimi/docs/repo-evaluation-contract.md | 1 + dokimi/dokimi/repos/l0/core.py | 29 ++++++++++++++----- dokimi/tests/test_repo_l0_cases.py | 6 ++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md b/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md index 5e34bce75..b9a4ae2f3 100644 --- a/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md +++ b/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md @@ -3,6 +3,6 @@ - **`repo scan` no longer fails a path the repository's `.gitignore` ignores** — guidance that names generated or ignored files had each such path reported `fail` as a rename, because its parent directory exists and the file is never tracked. A missing path, directory - or glob that a tracked `.gitignore` ignores is now `unresolved`, and the reason names the - `.gitignore` line. The rules follow gitignore(5), negation and nested files included; + (with or without its trailing slash) or glob that a tracked `.gitignore` ignores is now + `unresolved`, and the reason names the `.gitignore` line. The rules follow gitignore(5), negation and nested files included; `.git/info/exclude` and `core.excludesFile` are not part of the snapshot and are not read. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 84f58dcaf..082813954 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -893,6 +893,7 @@ path that a tracked `.gitignore` ignores is `unresolved` rather than `fail`, with the `.gitignore` line in the reason: guidance that warns about generated files names paths no snapshot holds. A directory counts as ignored when a file directly inside it would be, +a path without a trailing slash when it would be as a file or as a directory, and a glob when one file it names would be; `.git/info/exclude` and `core.excludesFile` belong to a clone, not the tree, and are not read. `ok_suffix` diff --git a/dokimi/dokimi/repos/l0/core.py b/dokimi/dokimi/repos/l0/core.py index b4e9899fa..6c8d1f6c5 100644 --- a/dokimi/dokimi/repos/l0/core.py +++ b/dokimi/dokimi/repos/l0/core.py @@ -21,7 +21,7 @@ from typing import Any from ..source import RepoTree, SourceError -from .gitignore import GitIgnore +from .gitignore import GitIgnore, Rule from .markdown import PYTHON_LANGS, SHELL_LANGS, STARLARK_LANGS, Fence, split # Files whose words are prose, not code: an identifier found only there is not in the code. @@ -86,6 +86,12 @@ def glob_regex(pat: str) -> re.Pattern[str]: return re.compile("".join(out) + r"\Z") +def _instances(pat: str) -> list[str]: + """Files a glob names, to ask a ``.gitignore`` about: two, so one re-included (``!x.cxx``) leaves the other.""" + stem = pat.replace("**/", "") + return [re.sub(r"\?", q, re.sub(r"\*+", star, stem)) for star, q in (("x", "x"), ("dokimi-probe", "_"))] + + def is_doc(path: str) -> bool: name = posixpath.basename(path) if name == "CMakeLists.txt" or name.startswith("requirements"): @@ -306,7 +312,7 @@ def resolve_path(self, file: str, raw: str) -> tuple[str, str] | None: for c in cands: parent = posixpath.dirname(c) if parent and parent in self.dir_set: - return self._missing(c, rel.endswith("/"), f"{parent}/ exists but {c} does not") + return self._missing([c], rel.endswith("/"), f"{parent}/ exists but {c} does not") hits = [x for x in self.by_name.get(posixpath.basename(target), []) if x.endswith("/" + target)] if hits: return "ok_suffix", f"below a directory the text does not name; {len(hits)} match(es), e.g. {hits[0]}" @@ -328,8 +334,7 @@ def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: for b in bases: d = posixpath.normpath(posixpath.join(b, parent)) if d in self.dir_set: - instance = posixpath.join(b, re.sub(r"\*+|\?", "x", pat.replace("**/", ""))) - return self._missing(instance, False, + return self._missing([posixpath.join(b, i) for i in _instances(pat)], False, f"{d}/ exists but nothing in it matches {posixpath.basename(pat)}") if parent: rx = glob_regex("*/" + pat) @@ -338,16 +343,26 @@ def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: return "ok_suffix", f"below a directory the text does not name; e.g. {hit}" return "unresolved", f"glob {pat} matches nothing in the tree; a build may create it" - def _missing(self, path: str, is_dir: bool, why: str) -> tuple[str, str]: + def _missing(self, paths: list[str], is_dir: bool, why: str) -> tuple[str, str]: """``fail`` for a path the tree lacks, unless a ``.gitignore`` ignores it: a build output. - A directory counts as ignored when a file directly inside it would be (``include/*``). + ``paths`` are the files a glob names, or the one path; any of them being ignored is enough. """ - rule = self.gitignore().ignored_by(posixpath.join(path, "file") if is_dir else path) + rule = next(filter(None, (self._ignored_by(p, is_dir) for p in paths)), None) if rule is None: return "fail", why return "unresolved", f"{why}; {rule.source}:{rule.line} (`{rule.pattern}`) ignores it, so it is not meant to be tracked" + def _ignored_by(self, path: str, is_dir: bool) -> Rule | None: + """A directory counts as ignored when a file directly inside it would be (``include/*``). + + A path without a trailing slash may still name a directory (``dist`` under ``dist/``). + """ + ig = self.gitignore() + if is_dir: + return ig.ignored_by(posixpath.join(path, "file")) + return ig.ignored_by(path) or ig.ignored_by(path, is_dir=True) + def gitignore(self) -> GitIgnore: def load() -> GitIgnore: self.preload([f for f in self.files if posixpath.basename(f) == ".gitignore"]) diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index aea925463..c3da8905c 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -81,6 +81,12 @@ def _shell(*lines: str) -> str: ("gitignored glob in a nested guidance file", "x", {"pkg/AGENTS.md": "`sub/*.cxx`\n", "pkg/sub/.gitignore": "*.cxx\n", "pkg/sub/a.pyx": ""}, {("path", "sub/*.cxx"): "unresolved"}), + ("gitignored directory written without a slash", "`pkg/dist`", + {"pkg/.gitignore": "dist/\n", "pkg/a.c": ""}, {("path", "pkg/dist"): "unresolved"}), + ("a whitelisting .gitignore keeps a missing file a rename", "`pkg/old.c`", + {"pkg/.gitignore": "*\n!*/\n!*.c\n", "pkg/a.c": ""}, {("path", "pkg/old.c"): "fail"}), + ("gitignored glob, one name it could stand for re-included", "`pkg/*.cxx`", + {"pkg/.gitignore": "*.cxx\n!x.cxx\n", "pkg/a.pyx": ""}, {("path", "pkg/*.cxx"): "unresolved"}), ("re-included by the .gitignore: a rename again", "`pkg/keep.h`", {"pkg/.gitignore": "*.h\n!keep.h\n", "pkg/a.c": ""}, {("path", "pkg/keep.h"): "fail"}), ("ignored elsewhere only", "`pkg/gen.h`", {"other/.gitignore": "*.h\n", "pkg/a.c": ""}, From b2444d8de25c856f2c9417490652cdc1e20d666a Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 27 Oct 2026 14:05:32 -0700 Subject: [PATCH 09/14] fix(repo): probe gitignored globs and directories by more than one name A glob was judged ignored by one made-up file, its wildcards filled in with `x`, so the verdict depended on that letter. `*.cxx` with `!x.cxx`, `[0-9]*.png`, `gen.h` and `gen_[a-c].h` all gave `fail` although git ignores real matches. The probe now fills the wildcards two ways (`x`, `dokimi-probe`) and adds each name an ignoring rule's last segment matches (`gen.h`, `0x.png`, `gen_a.h`). It keeps only the names the glob matches and asks the exact matcher about each. A glob written with a trailing slash is now checked as a directory. A missing directory was judged by the made-up name `/file`, which a whitelist (`*`, `!*/`, `!*.py`) always ignores, so a renamed directory read as `unresolved`. A directory now counts as ignored when git ignores it, or would ignore every name probed inside it: `file` and each name a `!` line re-includes. The reason now names the `.gitignore` line on the fallbacks too, where the parent directory is missing (`pkg/resources/cmake/`, `pkg/include/pkg/*.h`). It also no longer says the path is "not meant to be tracked", only that it is likely generated. The git oracle in the parity test sets HOME and XDG_CONFIG_HOME, so a developer's own core.excludesFile cannot change the answer. Both whitelist test cases now include `!.gitignore`; without it the `.gitignore` was never tracked and the cases checked nothing. Signed-off-by: Nikita Grigorian --- dokimi/docs/repo-evaluation-contract.md | 17 +++++--- dokimi/dokimi/repos/l0/core.py | 55 ++++++++++++++++++------- dokimi/dokimi/repos/l0/gitignore.py | 35 ++++++++++++++-- dokimi/tests/test_repo_l0_cases.py | 29 +++++++++++-- dokimi/tests/test_repo_l0_gitignore.py | 20 ++++++++- 5 files changed, 126 insertions(+), 30 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 082813954..6fb4e5903 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -890,12 +890,17 @@ at one of those bases (a rename or a deleted file), `ok_suffix` when it ends a longer path in the tree (a root file naming paths inside a subproject), and `unresolved` otherwise (a build output, a path inside a submodule). A missing path that a tracked `.gitignore` ignores is `unresolved` rather than `fail`, -with the `.gitignore` line in the reason: guidance that warns about generated -files names paths no snapshot holds. -A directory counts as ignored when a file directly inside it would be, -a path without a trailing slash when it would be as a file or as a directory, -and a glob when one file it names would be; -`.git/info/exclude` and `core.excludesFile` belong to a clone, not the tree, +with the `.gitignore` line in the reason (also for a path that was `unresolved` +already): guidance that warns about generated files names paths no snapshot +holds. A directory counts as ignored when git ignores it, or would ignore every +file probed directly inside it: a plain name and each name a `!` line +re-includes, so `include/*` ignores `include/` and a whitelist of `*`, `!*/`, +`!*.py` does not ignore a missing directory. A path without a trailing slash counts when it +would be as a file or as a directory, and a glob when one of the files it is +probed with would be: its wildcards filled in, and each name an ignoring line +matches that the glob matches too (`gen.h` for `pkg/g*.h`). The probe tries a +few names, not every name the glob matches. `.git/info/exclude` and +`core.excludesFile` belong to a clone, not the tree, and are not read. `ok_suffix` and `ok_basename` resolve a path but cannot see it renamed, which then reads as `unresolved`; guidance the gate must protect names its paths from the repository diff --git a/dokimi/dokimi/repos/l0/core.py b/dokimi/dokimi/repos/l0/core.py index 6c8d1f6c5..d8c05ca32 100644 --- a/dokimi/dokimi/repos/l0/core.py +++ b/dokimi/dokimi/repos/l0/core.py @@ -86,10 +86,12 @@ def glob_regex(pat: str) -> re.Pattern[str]: return re.compile("".join(out) + r"\Z") -def _instances(pat: str) -> list[str]: - """Files a glob names, to ask a ``.gitignore`` about: two, so one re-included (``!x.cxx``) leaves the other.""" - stem = pat.replace("**/", "") - return [re.sub(r"\?", q, re.sub(r"\*+", star, stem)) for star, q in (("x", "x"), ("dokimi-probe", "_"))] +_FILLS = ("x", "dokimi-probe") # what a `*` becomes in a probe of the files a glob names + + +def _fill(pat: str, fill: str) -> str: + """``pat`` with its wildcards filled in: ``**/`` as nothing, ``*`` as ``fill``, ``?`` as its first letter.""" + return re.sub(r"\?", fill[0], re.sub(r"\*+", fill, pat.replace("**/", ""))) def is_doc(path: str) -> bool: @@ -316,7 +318,7 @@ def resolve_path(self, file: str, raw: str) -> tuple[str, str] | None: hits = [x for x in self.by_name.get(posixpath.basename(target), []) if x.endswith("/" + target)] if hits: return "ok_suffix", f"below a directory the text does not name; {len(hits)} match(es), e.g. {hits[0]}" - return "unresolved", "tried " + ", ".join(cands) + return self._missing([cands[0]], rel.endswith("/"), "tried " + ", ".join(cands), "unresolved") def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: """A glob is judged like the plain path it generalises (see :meth:`resolve_path`).""" @@ -334,34 +336,55 @@ def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: for b in bases: d = posixpath.normpath(posixpath.join(b, parent)) if d in self.dir_set: - return self._missing([posixpath.join(b, i) for i in _instances(pat)], False, + probes = self._probes(posixpath.normpath(posixpath.join(b, pat))) + return self._missing(probes, rel.endswith("/"), f"{d}/ exists but nothing in it matches {posixpath.basename(pat)}") if parent: rx = glob_regex("*/" + pat) hit = next((x for x in every if rx.match(x)), None) if hit: return "ok_suffix", f"below a directory the text does not name; e.g. {hit}" - return "unresolved", f"glob {pat} matches nothing in the tree; a build may create it" + full = posixpath.normpath(posixpath.join(bases[0], pat)) + return self._missing(self._probes(full), rel.endswith("/"), + f"glob {pat} matches nothing in the tree; a build may create it", "unresolved") + + def _probes(self, full: str) -> list[str]: + """Files the glob ``full`` names, to ask the ``.gitignore`` about: a probe of a few names, not every one. + + Its wildcards filled in, and each name an ignoring rule matches that the glob matches too + (``gen.h`` for ``g*.h``); two fillings, so one re-included (``!x.cxx``) leaves the other. + """ + folder, name = posixpath.split(full) + rx = glob_regex(full) + paths = [posixpath.join(_fill(folder, fill), n) for fill in _FILLS + for n in (_fill(name, fill), *self.gitignore().names(fill))] + return [p for p in dict.fromkeys(paths) if rx.match(p)] - def _missing(self, paths: list[str], is_dir: bool, why: str) -> tuple[str, str]: - """``fail`` for a path the tree lacks, unless a ``.gitignore`` ignores it: a build output. + def _missing(self, paths: list[str], is_dir: bool, why: str, status: str = "fail") -> tuple[str, str]: + """``status`` for a path the tree lacks, unless a ``.gitignore`` ignores it: a build output. ``paths`` are the files a glob names, or the one path; any of them being ignored is enough. """ rule = next(filter(None, (self._ignored_by(p, is_dir) for p in paths)), None) if rule is None: - return "fail", why - return "unresolved", f"{why}; {rule.source}:{rule.line} (`{rule.pattern}`) ignores it, so it is not meant to be tracked" + return status, why + return "unresolved", (f"{why}; {rule.source}:{rule.line} (`{rule.pattern}`) ignores it, " + "so it is likely generated rather than tracked") def _ignored_by(self, path: str, is_dir: bool) -> Rule | None: - """A directory counts as ignored when a file directly inside it would be (``include/*``). + """The rule that ignores ``path``, read as a directory too: without a trailing slash it may + still name one (``dist`` under ``dist/``). - A path without a trailing slash may still name a directory (``dist`` under ``dist/``). + A directory also counts when every file probed directly inside it would be (``include/*``): + a plain name and each name a ``!`` line re-includes, so a whitelist (``*``, ``!*/``, + ``!*.py``) does not hide a renamed directory. """ ig = self.gitignore() - if is_dir: - return ig.ignored_by(posixpath.join(path, "file")) - return ig.ignored_by(path) or ig.ignored_by(path, is_dir=True) + rule = ig.ignored_by(path, is_dir=True) if is_dir else ig.ignored_by(path) or ig.ignored_by(path, is_dir=True) + if rule is not None: + return rule + inside = [ig.ignored_by(posixpath.join(path, n)) for n in ("file", *ig.names("x", negated=True))] + return inside[0] if all(inside) else None def gitignore(self) -> GitIgnore: def load() -> GitIgnore: diff --git a/dokimi/dokimi/repos/l0/gitignore.py b/dokimi/dokimi/repos/l0/gitignore.py index 372671bfd..bda4bd39d 100644 --- a/dokimi/dokimi/repos/l0/gitignore.py +++ b/dokimi/dokimi/repos/l0/gitignore.py @@ -33,6 +33,7 @@ class Rule: anchored: bool dir_only: bool negated: bool + name: str = "" # the last segment of the pattern, wildcards kept; "" for a line git cannot read def matches(self, path: str, is_dir: bool) -> bool: if self.dir_only and not is_dir: @@ -154,6 +155,28 @@ def _regex(pat: str) -> str: _NEVER = re.compile(r"(?!)") +_MEMBERS = "xad0_-.AZ9" # tried in turn for a ``[...]``: the first one it matches stands for it + + +def example(pat: str, fill: str) -> str | None: + """A name the one-segment ``pat`` matches: each ``*`` as ``fill``, each ``?`` as its first letter, + each ``[...]`` as a member; ``None`` when no candidate member fits a class.""" + out, i = [], 0 + while i < len(pat): + c = pat[i] + if c == "[": + atom, i = _bracket(pat, i) + out.append(next((m for m in fill[0] + _MEMBERS if re.fullmatch(atom, m)), "")) + if not out[-1]: + return None + continue + if c == "\\": + i += 1 + out.append(pat[i]) + else: + out.append({"*": fill, "?": fill[0]}.get(c, c)) + i += 1 + return "".join(out) def parse(source: str, text: str) -> list[Rule]: @@ -173,10 +196,10 @@ def parse(source: str, text: str) -> list[Rule]: continue anchored = "/" in pat try: - regex = re.compile(_regex(pat.lstrip("/")) + r"\Z") + regex, name = re.compile(_regex(pat.lstrip("/")) + r"\Z"), _split(pat.lstrip("/"))[-1] except _Invalid: - regex = _NEVER - rules.append(Rule(source, n, line, base, regex, anchored, dir_only, negated)) + regex, name = _NEVER, "" + rules.append(Rule(source, n, line, base, regex, anchored, dir_only, negated, name)) return rules @@ -191,6 +214,12 @@ def __init__(self, files: Iterable[str], read: Callable[[str], str]) -> None: def _last(self, path: str, is_dir: bool) -> Rule | None: return next((r for r in reversed(self.rules) if r.matches(path, is_dir)), None) + def names(self, fill: str, negated: bool = False) -> list[str]: + """A name the last segment of each ignoring rule matches (``gen_a.h`` for ``gen_[a-c].h``), + or with ``negated`` of each ``!`` rule.""" + found = (example(r.name, fill) for r in self.rules if r.name and r.negated == negated) + return list(dict.fromkeys(n for n in found if n)) + def ignored_by(self, path: str, is_dir: bool = False) -> Rule | None: """The rule that ignores ``path`` or one of its directories, or ``None`` when git would not.""" parts = posixpath.normpath(path).strip("/").split("/") diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index c3da8905c..1b8b799e1 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -84,9 +84,21 @@ def _shell(*lines: str) -> str: ("gitignored directory written without a slash", "`pkg/dist`", {"pkg/.gitignore": "dist/\n", "pkg/a.c": ""}, {("path", "pkg/dist"): "unresolved"}), ("a whitelisting .gitignore keeps a missing file a rename", "`pkg/old.c`", - {"pkg/.gitignore": "*\n!*/\n!*.c\n", "pkg/a.c": ""}, {("path", "pkg/old.c"): "fail"}), + {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.c\n", "pkg/a.c": ""}, {("path", "pkg/old.c"): "fail"}), ("gitignored glob, one name it could stand for re-included", "`pkg/*.cxx`", {"pkg/.gitignore": "*.cxx\n!x.cxx\n", "pkg/a.pyx": ""}, {("path", "pkg/*.cxx"): "unresolved"}), + ("gitignored directory contents, no trailing slash", "`pkg/include`", + {"pkg/.gitignore": "include/*\n", "pkg/a.c": ""}, {("path", "pkg/include"): "unresolved"}), + ("a whitelisting .gitignore keeps a missing directory a rename", "`pkg/olddir/`", + {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.py\n", "pkg/a.py": ""}, {("path", "pkg/olddir/"): "fail"}), + ("gitignored glob the fill-in name misses", "`img/*.png` `pkg/g*.h` `pkg/gen_*.h`", + {"img/.gitignore": "[0-9]*.png\n", "img/a.c": "", "pkg/.gitignore": "gen.h\ngen_[a-c].h\n", "pkg/a.c": ""}, + {("path", "img/*.png"): "unresolved", ("path", "pkg/g*.h"): "unresolved", + ("path", "pkg/gen_*.h"): "unresolved"}), + ("a glob no ignoring rule's names fit", "`pkg/*.c`", {"pkg/.gitignore": "gen.h\n", "pkg/a.h": ""}, + {("path", "pkg/*.c"): "fail"}), + ("gitignored directory glob", "`pkg/gen*/`", {"pkg/.gitignore": "gen*/\n", "pkg/a.c": ""}, + {("path", "pkg/gen*/"): "unresolved"}), ("re-included by the .gitignore: a rename again", "`pkg/keep.h`", {"pkg/.gitignore": "*.h\n!keep.h\n", "pkg/a.c": ""}, {("path", "pkg/keep.h"): "fail"}), ("ignored elsewhere only", "`pkg/gen.h`", {"other/.gitignore": "*.h\n", "pkg/a.c": ""}, @@ -302,9 +314,18 @@ def test_a_path_into_a_submodule_says_so(tmp_path): assert "inside the submodule third/dep" in payload["claims"][0]["why"] -def test_a_gitignored_path_names_the_line_that_ignores_it(tmp_path): - payload = _scan(tmp_path, "`pkg/include/`\n", {"pkg/.gitignore": "*.o\ninclude/*\n", "pkg/a.c": ""}) - assert "pkg/.gitignore:2 (`include/*`) ignores it" in payload["claims"][0]["why"] +@pytest.mark.parametrize("agents, files, line", [ + ("`pkg/include/`", {"pkg/.gitignore": "*.o\ninclude/*\n", "pkg/a.c": ""}, "pkg/.gitignore:2 (`include/*`)"), + # the parent directory is missing too + ("`pkg/resources/cmake/`", {".gitignore": "pkg/resources/cmake\n", "pkg/a.py": ""}, + ".gitignore:1 (`pkg/resources/cmake`)"), + ("`pkg/include/pkg/*.h`", {".gitignore": "pkg/include/pkg/*.h\n", "pkg/a.py": ""}, + ".gitignore:1 (`pkg/include/pkg/*.h`)"), +]) +def test_a_gitignored_path_names_the_line_that_ignores_it(tmp_path, agents, files, line): + claim = _scan(tmp_path, agents + "\n", files)["claims"][0] + assert claim["status"] == "unresolved" + assert f"{line} ignores it" in claim["why"], claim["why"] def test_no_claim_for_options_or_operators(tmp_path): diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py index 3473951d7..5ff7fe5a9 100644 --- a/dokimi/tests/test_repo_l0_gitignore.py +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -70,7 +70,9 @@ def _git_ignored(root: Path) -> set[str]: - env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"} + # HOME and XDG_CONFIG_HOME too: the default core.excludesFile is $XDG_CONFIG_HOME/git/ignore + env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", + "HOME": str(root), "XDG_CONFIG_HOME": str(root)} out = subprocess.run(["git", "-C", str(root), "check-ignore", "--no-index", "--", *PATHS], capture_output=True, text=True, env=env, check=False) assert out.returncode in (0, 1), out.stderr # 1: nothing ignored @@ -121,3 +123,19 @@ def test_many_stars_do_not_backtrack(): assert rules.ignored_by("pkg/" + "a" * 45) is None assert rules.ignored_by("pkg/" + "a" * 45 + "b") is not None assert time.perf_counter() - start < 1 + + +@pytest.mark.parametrize("line, name, path", [ + ("gen.h", "gen.h", "pkg/gen.h"), ("[0-9]*.png", "0x.png", "img/0x.png"), + ("gen_[a-c].h", "gen_a.h", "gen_a.h"), ("[!x]?.o", "ax.o", "ax.o"), ("[[:upper:]]*", "Ax", "Ax"), + ("\\*.h", "*.h", "*.h"), ("build/out*/", "outx", "build/outx/f"), +]) +def test_a_rule_names_a_file_it_ignores(line, name, path): + rules = GitIgnore({".gitignore": line}, lambda _: line) + assert rules.names("x") == [name] + assert rules.ignored_by(path) is not None + + +def test_a_class_no_candidate_fits_and_a_negation_name_nothing(): + line = "[!a-zA-Z0-9_.-]*.o\n!keep.o\n" + assert GitIgnore({".gitignore": line}, lambda _: line).names("x") == [] From 4dc53ec0fd6a1e5cecd8b7062f2b3025d66e6677 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Wed, 28 Oct 2026 10:42:43 -0700 Subject: [PATCH 10/14] fix(repo): keep a path a `!` line re-includes a rename, trim lines as git does A missing path the last matching line re-includes (`!/setup.py`, `!/olddir/` under `*`, `!*/`) was probed as a directory, and the files probed inside it were all ignored, so it read `unresolved`. The inside probe now runs only for a directory no `!` line re-includes. .gitignore lines are read as git reads them: split at `\n` only with a `\r` before it dropped, a byte-order mark skipped, and only unescaped trailing spaces trimmed (a tab stays; `bs\\ ` loses its space). The git oracle reads paths with -z so such names compare unquoted. The reason for a path or glob tried at several bases now names the .gitignore line at any of them, not only at the root. Tests pin `_`, `-` and `.` as class members, and the changelog says fail counts can change for the same repository. Signed-off-by: Nikita Grigorian --- ...50000_ndgrigorian_scan-gitignored-paths.md | 2 ++ dokimi/docs/repo-evaluation-contract.md | 9 ++++---- dokimi/dokimi/repos/l0/core.py | 20 ++++++++-------- dokimi/dokimi/repos/l0/gitignore.py | 23 +++++++++++++++++-- dokimi/tests/test_repo_l0_cases.py | 10 +++++++- dokimi/tests/test_repo_l0_gitignore.py | 11 ++++++--- 6 files changed, 56 insertions(+), 19 deletions(-) diff --git a/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md b/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md index b9a4ae2f3..03b380130 100644 --- a/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md +++ b/dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md @@ -6,3 +6,5 @@ (with or without its trailing slash) or glob that a tracked `.gitignore` ignores is now `unresolved`, and the reason names the `.gitignore` line. The rules follow gitignore(5), negation and nested files included; `.git/info/exclude` and `core.excludesFile` are not part of the snapshot and are not read. + Some paths an earlier scan reported `fail` are now `unresolved`, so the fail count and the L0 + gate result can differ from a scan of the same repository made before this change. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 6fb4e5903..5daacfffc 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -892,10 +892,11 @@ longer path in the tree (a root file naming paths inside a subproject), and path that a tracked `.gitignore` ignores is `unresolved` rather than `fail`, with the `.gitignore` line in the reason (also for a path that was `unresolved` already): guidance that warns about generated files names paths no snapshot -holds. A directory counts as ignored when git ignores it, or would ignore every -file probed directly inside it: a plain name and each name a `!` line -re-includes, so `include/*` ignores `include/` and a whitelist of `*`, `!*/`, -`!*.py` does not ignore a missing directory. A path without a trailing slash counts when it +holds. A directory counts as ignored when git ignores it, or, when no `!` line +re-includes the directory itself, would ignore every file probed directly inside +it: a plain name and each name a `!` line re-includes, so `include/*` ignores +`include/` and a whitelist of `*`, `!*/`, `!*.py` does not ignore a missing +directory, nor `!/setup.py` a missing `setup.py`. A path without a trailing slash counts when it would be as a file or as a directory, and a glob when one of the files it is probed with would be: its wildcards filled in, and each name an ignoring line matches that the glob matches too (`gen.h` for `pkg/g*.h`). The probe tries a diff --git a/dokimi/dokimi/repos/l0/core.py b/dokimi/dokimi/repos/l0/core.py index d8c05ca32..87c4398af 100644 --- a/dokimi/dokimi/repos/l0/core.py +++ b/dokimi/dokimi/repos/l0/core.py @@ -318,7 +318,7 @@ def resolve_path(self, file: str, raw: str) -> tuple[str, str] | None: hits = [x for x in self.by_name.get(posixpath.basename(target), []) if x.endswith("/" + target)] if hits: return "ok_suffix", f"below a directory the text does not name; {len(hits)} match(es), e.g. {hits[0]}" - return self._missing([cands[0]], rel.endswith("/"), "tried " + ", ".join(cands), "unresolved") + return self._missing(cands, rel.endswith("/"), "tried " + ", ".join(cands), "unresolved") def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: """A glob is judged like the plain path it generalises (see :meth:`resolve_path`).""" @@ -344,8 +344,8 @@ def _glob(self, bases: list[str], rel: str) -> tuple[str, str]: hit = next((x for x in every if rx.match(x)), None) if hit: return "ok_suffix", f"below a directory the text does not name; e.g. {hit}" - full = posixpath.normpath(posixpath.join(bases[0], pat)) - return self._missing(self._probes(full), rel.endswith("/"), + probes = [p for b in bases for p in self._probes(posixpath.normpath(posixpath.join(b, pat)))] + return self._missing(probes, rel.endswith("/"), f"glob {pat} matches nothing in the tree; a build may create it", "unresolved") def _probes(self, full: str) -> list[str]: @@ -363,7 +363,8 @@ def _probes(self, full: str) -> list[str]: def _missing(self, paths: list[str], is_dir: bool, why: str, status: str = "fail") -> tuple[str, str]: """``status`` for a path the tree lacks, unless a ``.gitignore`` ignores it: a build output. - ``paths`` are the files a glob names, or the one path; any of them being ignored is enough. + ``paths`` are the files a glob names, or the path at each base it is tried from; any of them + being ignored is enough. """ rule = next(filter(None, (self._ignored_by(p, is_dir) for p in paths)), None) if rule is None: @@ -375,13 +376,14 @@ def _ignored_by(self, path: str, is_dir: bool) -> Rule | None: """The rule that ignores ``path``, read as a directory too: without a trailing slash it may still name one (``dist`` under ``dist/``). - A directory also counts when every file probed directly inside it would be (``include/*``): - a plain name and each name a ``!`` line re-includes, so a whitelist (``*``, ``!*/``, - ``!*.py``) does not hide a renamed directory. + A directory no ``!`` line re-includes also counts when every file probed directly inside it + would be (``include/*``): a plain name and each name a ``!`` line re-includes, so a whitelist + (``*``, ``!*/``, ``!*.py``) does not hide a renamed directory, nor ``!/setup.py`` a renamed file. """ ig = self.gitignore() - rule = ig.ignored_by(path, is_dir=True) if is_dir else ig.ignored_by(path) or ig.ignored_by(path, is_dir=True) - if rule is not None: + readings = (True,) if is_dir else (False, True) + rule = next(filter(None, (ig.ignored_by(path, d) for d in readings)), None) + if rule is not None or any(ig.reincludes(path, d) for d in readings): return rule inside = [ig.ignored_by(posixpath.join(path, n)) for n in ("file", *ig.names("x", negated=True))] return inside[0] if all(inside) else None diff --git a/dokimi/dokimi/repos/l0/gitignore.py b/dokimi/dokimi/repos/l0/gitignore.py index bda4bd39d..0cff1d9d6 100644 --- a/dokimi/dokimi/repos/l0/gitignore.py +++ b/dokimi/dokimi/repos/l0/gitignore.py @@ -179,11 +179,25 @@ def example(pat: str, fill: str) -> str | None: return "".join(out) +def _trim(line: str) -> str: + """``line`` without its trailing spaces, as git trims it: a tab stays, and so does a space after a ``\\``.""" + end, i = len(line), 0 + while i < len(line): + if line[i] != " ": + end = len(line) + i += line[i] == "\\" # the escaped character, a space or a second backslash + elif end == len(line): + end = i + i += 1 + return line[:end] + + def parse(source: str, text: str) -> list[Rule]: base = posixpath.dirname(source) rules = [] - for n, raw in enumerate(text.splitlines(), 1): - line = re.sub(r"(? list[str]: found = (example(r.name, fill) for r in self.rules if r.name and r.negated == negated) return list(dict.fromkeys(n for n in found if n)) + def reincludes(self, path: str, is_dir: bool = False) -> bool: + """Whether the last line that matches ``path`` itself is a ``!`` line.""" + rule = self._last(posixpath.normpath(path).strip("/"), is_dir) + return rule is not None and rule.negated + def ignored_by(self, path: str, is_dir: bool = False) -> Rule | None: """The rule that ignores ``path`` or one of its directories, or ``None`` when git would not.""" parts = posixpath.normpath(path).strip("/").split("/") diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index 1b8b799e1..458e5fb8b 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -85,6 +85,9 @@ def _shell(*lines: str) -> str: {"pkg/.gitignore": "dist/\n", "pkg/a.c": ""}, {("path", "pkg/dist"): "unresolved"}), ("a whitelisting .gitignore keeps a missing file a rename", "`pkg/old.c`", {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.c\n", "pkg/a.c": ""}, {("path", "pkg/old.c"): "fail"}), + ("an anchored re-include under a whitelist keeps a missing path a rename", "`pkg/setup.py` `pkg/olddir/`", + {"pkg/.gitignore": "*\n!*/\n!/.gitignore\n!/setup.py\n!/olddir/\n", "pkg/keep.txt": ""}, + {("path", "pkg/setup.py"): "fail", ("path", "pkg/olddir/"): "fail"}), ("gitignored glob, one name it could stand for re-included", "`pkg/*.cxx`", {"pkg/.gitignore": "*.cxx\n!x.cxx\n", "pkg/a.pyx": ""}, {("path", "pkg/*.cxx"): "unresolved"}), ("gitignored directory contents, no trailing slash", "`pkg/include`", @@ -321,9 +324,14 @@ def test_a_path_into_a_submodule_says_so(tmp_path): ".gitignore:1 (`pkg/resources/cmake`)"), ("`pkg/include/pkg/*.h`", {".gitignore": "pkg/include/pkg/*.h\n", "pkg/a.py": ""}, ".gitignore:1 (`pkg/include/pkg/*.h`)"), + # tried from the root first; the guidance file's own directory is the one ignored + ("x", {"pkg/AGENTS.md": "`gen/out.h`\n", "pkg/.gitignore": "/gen/\n", "pkg/a.c": ""}, + "pkg/.gitignore:1 (`/gen/`)"), + ("x", {"pkg/AGENTS.md": "`resources/*.h`\n", "pkg/.gitignore": "/resources/*.h\n", "pkg/a.c": ""}, + "pkg/.gitignore:1 (`/resources/*.h`)"), ]) def test_a_gitignored_path_names_the_line_that_ignores_it(tmp_path, agents, files, line): - claim = _scan(tmp_path, agents + "\n", files)["claims"][0] + claim = next(c for c in _scan(tmp_path, agents + "\n", files)["claims"] if c["kind"] == "path") assert claim["status"] == "unresolved" assert f"{line} ignores it" in claim["why"], claim["why"] diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py index 5ff7fe5a9..bf426ffa2 100644 --- a/dokimi/tests/test_repo_l0_gitignore.py +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -44,9 +44,12 @@ "[^x]z.bin", "[a-]w", "[[:digit:]]9.d", + "tab\t", # git trims spaces, not tabs + "bs\\\\ ", # an escaped backslash does not keep the space after it "", ]), "sub/.gitignore": "!*.o\nlocal\n", + "crlf/.gitignore": "\ufeffbom\r\nlone\rcr\n", # git skips the mark, splits at \n only, drops \r before it } PATHS = [ @@ -66,6 +69,7 @@ "d/e/f", "dbe/f", "]q", "aq", "sub/local", "local", "sub/deeper/local", "!bang", "sp ", "sp", "yz.bin", "xz.bin", "-w", "aw", "99.d", + "tab\t", "tab", "bs\\", "bs\\ ", "crlf/bom", "crlf/lone\rcr", "crlf/lone", "crlf/cr", ] @@ -73,10 +77,10 @@ def _git_ignored(root: Path) -> set[str]: # HOME and XDG_CONFIG_HOME too: the default core.excludesFile is $XDG_CONFIG_HOME/git/ignore env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", "HOME": str(root), "XDG_CONFIG_HOME": str(root)} - out = subprocess.run(["git", "-C", str(root), "check-ignore", "--no-index", "--", *PATHS], - capture_output=True, text=True, env=env, check=False) + out = subprocess.run(["git", "-C", str(root), "check-ignore", "--no-index", "--stdin", "-z"], + input="\0".join(PATHS).encode(), capture_output=True, env=env, check=False) assert out.returncode in (0, 1), out.stderr # 1: nothing ignored - return set(out.stdout.split("\n")) - {""} + return set(out.stdout.decode().split("\0")) - {""} # -z and bytes: names unquoted, a \r kept def test_the_matcher_ignores_what_git_ignores(tmp_path): @@ -129,6 +133,7 @@ def test_many_stars_do_not_backtrack(): ("gen.h", "gen.h", "pkg/gen.h"), ("[0-9]*.png", "0x.png", "img/0x.png"), ("gen_[a-c].h", "gen_a.h", "gen_a.h"), ("[!x]?.o", "ax.o", "ax.o"), ("[[:upper:]]*", "Ax", "Ax"), ("\\*.h", "*.h", "*.h"), ("build/out*/", "outx", "build/outx/f"), + ("[_]gen.h", "_gen.h", "_gen.h"), ("[-]gen.h", "-gen.h", "-gen.h"), ("[.]gen.h", ".gen.h", ".gen.h"), ]) def test_a_rule_names_a_file_it_ignores(line, name, path): rules = GitIgnore({".gitignore": line}, lambda _: line) From 51d5220fe465384fd956e8029861dc1cac1200e9 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Thu, 29 Oct 2026 07:31:51 -0700 Subject: [PATCH 11/14] test: check each API shape, pin git's ignore set, split the names test - The API-name guard test now asserts each shape catches its own name (dotted call, bare call, keyword argument), so dropping or weakening one shape fails it. - The gitignore parity test compares the matcher with a pinned set of what `git check-ignore` printed, so it runs without git. A second test, skipped without git or on Windows, checks git still prints that set. It creates only the .gitignore files (--no-index matches names), runs `git init` with --template= and the isolated env. - The names() test is split into a class no candidate fits and a `!` rule, each with a readable line as a positive control, and checks names(..., negated=True). Signed-off-by: Nikita Grigorian --- dokimi/tests/test_agent_profiles.py | 5 ++- dokimi/tests/test_repo_l0_gitignore.py | 54 +++++++++++++++++++------- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/dokimi/tests/test_agent_profiles.py b/dokimi/tests/test_agent_profiles.py index ed9863df2..8d26cf12c 100644 --- a/dokimi/tests/test_agent_profiles.py +++ b/dokimi/tests/test_agent_profiles.py @@ -444,8 +444,9 @@ def test_the_api_name_check_would_actually_fire(): "Call asnumpy() once at the edge, pass usm_type='device' and sycl_queue=q to " "dpnp.tensor.empty(), and set dtype=float32." ) - leaked = {m for shape in API_SHAPES for m in re.findall(shape, contaminated)} - assert leaked, "the API-name guard is vacuous" + # One name per shape: a dotted call, a bare call, a keyword argument. Each needs its own. + for shape, name in zip(API_SHAPES, ("tensor.empty(", "asnumpy()", "usm_type='"), strict=True): + assert name in re.findall(shape, contaminated), f"{shape} no longer catches {name}" def test_a_persona_that_paraphrases_its_skill_is_detected(): diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py index bf426ffa2..849035e4f 100644 --- a/dokimi/tests/test_repo_l0_gitignore.py +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -6,7 +6,9 @@ from __future__ import annotations import os +import shutil import subprocess +import sys import time from pathlib import Path @@ -73,28 +75,43 @@ ] +# What `git check-ignore` printed for PATHS, so the matcher is checked on a host without git too. +GIT_IGNORED = { + "x.o", "deep/x.o", "build/a/b.c", "src/build/x.c", "build/keep.txt", "top.txt", + "docs/gen", "docs/a/b/gen", "cache", "x/cache/y.txt", "logs/a/b.txt", "a/b", "a/x/y/b", + "data/x.csv", "#hash", "a.tmp", "ay.bin", "include/x.h", "z.h", "1.d", "]x.e", "ax", "bx", + "dbe/f", "]q", "aq", "sub/local", "sub/deeper/local", "!bang", "sp ", "yz.bin", "-w", "aw", + "99.d", "tab\t", "bs\\", "crlf/bom", "crlf/lone\rcr", +} + + def _git_ignored(root: Path) -> set[str]: - # HOME and XDG_CONFIG_HOME too: the default core.excludesFile is $XDG_CONFIG_HOME/git/ignore + # HOME and XDG_CONFIG_HOME too: the default core.excludesFile is $XDG_CONFIG_HOME/git/ignore, + # and git before 2.32 reads $HOME/.gitconfig whatever GIT_CONFIG_GLOBAL says env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", "HOME": str(root), "XDG_CONFIG_HOME": str(root)} + # No template: a host init.templateDir could bring info/exclude rules + subprocess.run(["git", "init", "-q", "--template=", str(root)], check=True, env=env) + for rel, text in IGNORES.items(): + (root / rel).parent.mkdir(parents=True, exist_ok=True) + (root / rel).write_text(text, encoding="utf-8") + # --no-index matches the names alone, so PATHS (a tab, a \r, a trailing space) need no files out = subprocess.run(["git", "-C", str(root), "check-ignore", "--no-index", "--stdin", "-z"], input="\0".join(PATHS).encode(), capture_output=True, env=env, check=False) assert out.returncode in (0, 1), out.stderr # 1: nothing ignored return set(out.stdout.decode().split("\0")) - {""} # -z and bytes: names unquoted, a \r kept -def test_the_matcher_ignores_what_git_ignores(tmp_path): - subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) - for rel, text in IGNORES.items(): - (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) - (tmp_path / rel).write_text(text, encoding="utf-8") - for rel in PATHS: - (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) - (tmp_path / rel).write_text("", encoding="utf-8") - +def test_the_matcher_ignores_what_git_ignores(): rules = GitIgnore(IGNORES, IGNORES.__getitem__) - ours = {p for p in PATHS if rules.ignored_by(p) is not None} - assert ours == _git_ignored(tmp_path) + assert {p for p in PATHS if rules.ignored_by(p) is not None} == GIT_IGNORED + + +# Windows git may read a \ in a path as a separator, and init sets core.ignorecase there +@pytest.mark.skipif(shutil.which("git") is None or sys.platform == "win32", + reason="needs a POSIX git") +def test_git_still_ignores_what_the_matcher_is_checked_against(tmp_path): + assert _git_ignored(tmp_path) == GIT_IGNORED @pytest.mark.parametrize("path, source, line", [ @@ -141,6 +158,13 @@ def test_a_rule_names_a_file_it_ignores(line, name, path): assert rules.ignored_by(path) is not None -def test_a_class_no_candidate_fits_and_a_negation_name_nothing(): - line = "[!a-zA-Z0-9_.-]*.o\n!keep.o\n" - assert GitIgnore({".gitignore": line}, lambda _: line).names("x") == [] +def test_a_class_no_candidate_fits_names_nothing(): + line = "[!a-zA-Z0-9_.-]*.o\ngen.h\n" + assert GitIgnore({".gitignore": line}, lambda _: line).names("x") == ["gen.h"] + + +def test_a_negation_is_named_only_when_asked_for(): + line = "gen.h\n!keep.o\n" + rules = GitIgnore({".gitignore": line}, lambda _: line) + assert rules.names("x") == ["gen.h"] + assert rules.names("x", negated=True) == ["keep.o"] From e17d3edc62a5d243b43b4a1564aada0cbc82d1c8 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Thu, 29 Oct 2026 08:02:36 -0700 Subject: [PATCH 12/14] fix(repo): a `!*/` re-include no longer hides an ignored directory A directory that a `!*/`-style line re-includes, with every file probed directly inside it still ignored (`out/*` or `tmp/**` with `!*/`), read `fail`: the probe of its contents ran only when no `!` line matched the directory. Now only a `!` line that names the path (`!/olddir/`, `!/setup.py`, `!*.py`) skips the probe; one whose last segment is all stars re-includes every directory and does not. `GitIgnore.reincludes` becomes `reincluded_by` and returns the rule. Under a whitelist (`*`, `!*/`, `!*.py`) a missing path written without a slash stays ignored as a file (`pkg/Makefile`, and `pkg/olddir`): the contract now says a directory needs its trailing slash there to be checked as a rename. New CASES rows pin the verdicts a mutation run showed were untested: the file reading of a path without a slash, `include/*` with one re-included file, and a directory glob under a whitelist. Signed-off-by: Nikita Grigorian --- dokimi/docs/repo-evaluation-contract.md | 13 ++++++++----- dokimi/dokimi/repos/l0/core.py | 11 +++++++---- dokimi/dokimi/repos/l0/gitignore.py | 6 +++--- dokimi/tests/test_repo_l0_cases.py | 11 +++++++++++ 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 44666fba5..b49075ecb 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1062,11 +1062,14 @@ path that a tracked `.gitignore` ignores is `unresolved` rather than `fail`, with the `.gitignore` line in the reason (also for a path that was `unresolved` already): guidance that warns about generated files names paths no snapshot holds. A directory counts as ignored when git ignores it, or, when no `!` line -re-includes the directory itself, would ignore every file probed directly inside -it: a plain name and each name a `!` line re-includes, so `include/*` ignores -`include/` and a whitelist of `*`, `!*/`, `!*.py` does not ignore a missing -directory, nor `!/setup.py` a missing `setup.py`. A path without a trailing slash counts when it -would be as a file or as a directory, and a glob when one of the files it is +names the directory itself (`!*/` names every directory, so it does not count), +would ignore every file probed directly inside it: a plain name and each name a +`!` line re-includes. So `include/*` ignores `include/`, and so does `out/*` with +`!*/`, while a whitelist of `*`, `!*/`, `!*.py` does not ignore a missing +`olddir/`, nor `!/setup.py` a missing `setup.py`. A path without a trailing slash counts when it +would be as a file or as a directory: under that whitelist `*` ignores a missing +`olddir` read as a file, so a directory needs its trailing slash to be checked +as a rename. A glob counts when one of the files it is probed with would be: its wildcards filled in, and each name an ignoring line matches that the glob matches too (`gen.h` for `pkg/g*.h`). The probe tries a few names, not every name the glob matches. `.git/info/exclude` and diff --git a/dokimi/dokimi/repos/l0/core.py b/dokimi/dokimi/repos/l0/core.py index f790547bc..3a8b2fff4 100644 --- a/dokimi/dokimi/repos/l0/core.py +++ b/dokimi/dokimi/repos/l0/core.py @@ -376,15 +376,18 @@ def _ignored_by(self, path: str, is_dir: bool) -> Rule | None: """The rule that ignores ``path``, read as a directory too: without a trailing slash it may still name one (``dist`` under ``dist/``). - A directory no ``!`` line re-includes also counts when every file probed directly inside it - would be (``include/*``): a plain name and each name a ``!`` line re-includes, so a whitelist - (``*``, ``!*/``, ``!*.py``) does not hide a renamed directory, nor ``!/setup.py`` a renamed file. + A path no ``!`` line names (``!/setup.py``, ``!*.py``; ``!*/`` names every directory, not this + one) also counts when every file probed directly inside it would be (``include/*``, or ``out/*`` + with ``!*/``): a plain name and each name a ``!`` line re-includes, so a whitelist (``*``, + ``!*/``, ``!*.py``) does not hide a renamed directory. """ ig = self.gitignore() readings = (True,) if is_dir else (False, True) rule = next(filter(None, (ig.ignored_by(path, d) for d in readings)), None) - if rule is not None or any(ig.reincludes(path, d) for d in readings): + if rule is not None: return rule + if any(r and r.name.strip("*") for r in (ig.reincluded_by(path, d) for d in readings)): + return None inside = [ig.ignored_by(posixpath.join(path, n)) for n in ("file", *ig.names("x", negated=True))] return inside[0] if all(inside) else None diff --git a/dokimi/dokimi/repos/l0/gitignore.py b/dokimi/dokimi/repos/l0/gitignore.py index 0cff1d9d6..09dcb0e55 100644 --- a/dokimi/dokimi/repos/l0/gitignore.py +++ b/dokimi/dokimi/repos/l0/gitignore.py @@ -234,10 +234,10 @@ def names(self, fill: str, negated: bool = False) -> list[str]: found = (example(r.name, fill) for r in self.rules if r.name and r.negated == negated) return list(dict.fromkeys(n for n in found if n)) - def reincludes(self, path: str, is_dir: bool = False) -> bool: - """Whether the last line that matches ``path`` itself is a ``!`` line.""" + def reincluded_by(self, path: str, is_dir: bool = False) -> Rule | None: + """The ``!`` line that matches ``path`` itself last, or ``None`` when the last match ignores it or none does.""" rule = self._last(posixpath.normpath(path).strip("/"), is_dir) - return rule is not None and rule.negated + return rule if rule is not None and rule.negated else None def ignored_by(self, path: str, is_dir: bool = False) -> Rule | None: """The rule that ignores ``path`` or one of its directories, or ``None`` when git would not.""" diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index c80e3295c..f4de35539 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -107,6 +107,17 @@ def _wf(matrix: str) -> dict[str, str]: {"pkg/.gitignore": "include/*\n", "pkg/a.c": ""}, {("path", "pkg/include"): "unresolved"}), ("a whitelisting .gitignore keeps a missing directory a rename", "`pkg/olddir/`", {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.py\n", "pkg/a.py": ""}, {("path", "pkg/olddir/"): "fail"}), + ("a whitelisting .gitignore keeps a missing directory glob a rename", "`pkg/old*/`", + {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.py\n", "pkg/a.py": ""}, {("path", "pkg/old*/"): "fail"}), + ("under a whitelist a path without a slash is ignored as a file", "`pkg/Makefile` `pkg/olddir`", + {"pkg/.gitignore": "*\n!*/\n!.gitignore\n!*.py\n", "pkg/a.py": ""}, + {("path", "pkg/Makefile"): "unresolved", ("path", "pkg/olddir"): "unresolved"}), + ("a directory re-included by `!*/`, its files still ignored", "`pkg/out/` `pkg/tmp/`", + {"pkg/.gitignore": "out/*\ntmp/**\n!*/\n", "pkg/a.c": ""}, + {("path", "pkg/out/"): "unresolved", ("path", "pkg/tmp/"): "unresolved"}), + ("a directory with one file re-included is a rename", "`pkg/include/` `pkg/include`", + {"pkg/.gitignore": "include/*\n!include/keep.h\n", "pkg/a.c": ""}, + {("path", "pkg/include/"): "fail", ("path", "pkg/include"): "fail"}), ("gitignored glob the fill-in name misses", "`img/*.png` `pkg/g*.h` `pkg/gen_*.h`", {"img/.gitignore": "[0-9]*.png\n", "img/a.c": "", "pkg/.gitignore": "gen.h\ngen_[a-c].h\n", "pkg/a.c": ""}, {("path", "img/*.png"): "unresolved", ("path", "pkg/g*.h"): "unresolved", From c6db40630b85b062bf4dc8f2d26aa6e27e88deec Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Thu, 29 Oct 2026 09:58:22 -0700 Subject: [PATCH 13/14] fix(repo): a glob of only wildcards is not probed with rule names `_probes` added a name from every ignoring rule and kept those the glob matched, so a last segment such as `*` or `*_*` matched an unrelated rule (`_skbuild`): `maintenance/*` named `.gitignore:1 (_skbuild)` in its reason, and `pkg/*_*` with `pkg/` present went from `fail` to `unresolved`. Rule names now count only when a literal starts or ends the segment (`g*.h`, `*.png`, `gen*`); the filled-in names are still tried. Signed-off-by: Nikita Grigorian --- dokimi/docs/repo-evaluation-contract.md | 6 ++++-- dokimi/dokimi/repos/l0/core.py | 5 ++++- dokimi/tests/test_repo_l0_cases.py | 13 +++++++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index b49075ecb..5a05bba8c 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1071,8 +1071,10 @@ would be as a file or as a directory: under that whitelist `*` ignores a missing `olddir` read as a file, so a directory needs its trailing slash to be checked as a rename. A glob counts when one of the files it is probed with would be: its wildcards filled in, and each name an ignoring line -matches that the glob matches too (`gen.h` for `pkg/g*.h`). The probe tries a -few names, not every name the glob matches. `.git/info/exclude` and +matches that the glob matches too (`gen.h` for `pkg/g*.h`), when a literal +starts or ends the glob's last segment: `maintenance/*` or `pkg/*_*` would match +the name of any rule (`_skbuild`), which says nothing about the glob. The probe +tries a few names, not every name the glob matches. `.git/info/exclude` and `core.excludesFile` belong to a clone, not the tree, and are not read. `ok_suffix` and `ok_basename` resolve a path but cannot see it renamed, which then reads as diff --git a/dokimi/dokimi/repos/l0/core.py b/dokimi/dokimi/repos/l0/core.py index 3a8b2fff4..043b841dc 100644 --- a/dokimi/dokimi/repos/l0/core.py +++ b/dokimi/dokimi/repos/l0/core.py @@ -353,11 +353,14 @@ def _probes(self, full: str) -> list[str]: Its wildcards filled in, and each name an ignoring rule matches that the glob matches too (``gen.h`` for ``g*.h``); two fillings, so one re-included (``!x.cxx``) leaves the other. + Rule names count only for a last segment a literal starts or ends: ``*`` or ``*_*`` matches + the name of an unrelated rule (``_skbuild``), which is no evidence the glob is ignored. """ folder, name = posixpath.split(full) rx = glob_regex(full) + named = name[:1] not in "*?[" or name[-1:] not in "*?]" paths = [posixpath.join(_fill(folder, fill), n) for fill in _FILLS - for n in (_fill(name, fill), *self.gitignore().names(fill))] + for n in (_fill(name, fill), *(self.gitignore().names(fill) if named else ()))] return [p for p in dict.fromkeys(paths) if rx.match(p)] def _missing(self, paths: list[str], is_dir: bool, why: str, status: str = "fail") -> tuple[str, str]: diff --git a/dokimi/tests/test_repo_l0_cases.py b/dokimi/tests/test_repo_l0_cases.py index f4de35539..c11f0fbea 100644 --- a/dokimi/tests/test_repo_l0_cases.py +++ b/dokimi/tests/test_repo_l0_cases.py @@ -118,10 +118,12 @@ def _wf(matrix: str) -> dict[str, str]: ("a directory with one file re-included is a rename", "`pkg/include/` `pkg/include`", {"pkg/.gitignore": "include/*\n!include/keep.h\n", "pkg/a.c": ""}, {("path", "pkg/include/"): "fail", ("path", "pkg/include"): "fail"}), - ("gitignored glob the fill-in name misses", "`img/*.png` `pkg/g*.h` `pkg/gen_*.h`", + ("gitignored glob the fill-in name misses", "`img/*.png` `pkg/g*.h` `pkg/gen_*.h` `pkg/gen*`", {"img/.gitignore": "[0-9]*.png\n", "img/a.c": "", "pkg/.gitignore": "gen.h\ngen_[a-c].h\n", "pkg/a.c": ""}, {("path", "img/*.png"): "unresolved", ("path", "pkg/g*.h"): "unresolved", - ("path", "pkg/gen_*.h"): "unresolved"}), + ("path", "pkg/gen_*.h"): "unresolved", ("path", "pkg/gen*"): "unresolved"}), + ("a glob of wildcards and no literal end is not probed with rule names", "`pkg/*` `pkg/*_*`", + {"pkg/.gitignore": "_skbuild\n", "pkg/a.py": ""}, {("path", "pkg/*"): "ok", ("path", "pkg/*_*"): "fail"}), ("a glob no ignoring rule's names fit", "`pkg/*.c`", {"pkg/.gitignore": "gen.h\n", "pkg/a.h": ""}, {("path", "pkg/*.c"): "fail"}), ("gitignored directory glob", "`pkg/gen*/`", {"pkg/.gitignore": "gen*/\n", "pkg/a.c": ""}, @@ -568,6 +570,13 @@ def test_a_path_into_a_submodule_says_so(tmp_path): assert "inside the submodule third/dep" in payload["claims"][0]["why"] +def test_a_glob_an_unrelated_rule_spells_names_no_rule(tmp_path): + files = {".gitignore": "_skbuild\n", "pkg/a.py": ""} + claim = _scan(tmp_path, "`maintenance/*`\n", files)["claims"][0] + assert (claim["status"], claim["why"]) == ("unresolved", "glob maintenance/* matches nothing in the tree; " + "a build may create it") + + @pytest.mark.parametrize("agents, files, line", [ ("`pkg/include/`", {"pkg/.gitignore": "*.o\ninclude/*\n", "pkg/a.c": ""}, "pkg/.gitignore:2 (`include/*`)"), # the parent directory is missing too From 112323143024b960824853e702aa4b9c07e19695 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Thu, 29 Oct 2026 11:13:21 -0700 Subject: [PATCH 14/14] test: a line with no pattern is no rule, a tree without .gitignore ignores nothing Covers the empty-input cases the matcher skipped: a blank or whitespace-only line, and lines that reduce to no pattern ("/", "//", "!", "!/"), each of which git check-ignore also ignores nothing for. Also a tree with no .gitignore, where no file is read. Signed-off-by: Nikita Grigorian --- dokimi/tests/test_repo_l0_gitignore.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dokimi/tests/test_repo_l0_gitignore.py b/dokimi/tests/test_repo_l0_gitignore.py index 849035e4f..019c9e9b6 100644 --- a/dokimi/tests/test_repo_l0_gitignore.py +++ b/dokimi/tests/test_repo_l0_gitignore.py @@ -136,6 +136,21 @@ def test_a_line_git_cannot_read_matches_nothing(line, path): assert GitIgnore({".gitignore": line}, lambda _: line).ignored_by(path) is None +# git check-ignore ignores nothing for each of these lines: none leaves a pattern +@pytest.mark.parametrize("line", ["", " ", "/", "//", "!", "!/"]) +def test_a_line_with_no_pattern_is_no_rule(line): + text = f"gen.h\n{line}\n" + rules = GitIgnore({".gitignore": text}, lambda _: text) + assert [r.line for r in rules.rules] == [1] + assert rules.ignored_by("a/b") is None and rules.ignored_by("a", is_dir=True) is None + assert rules.reincluded_by("gen.h") is None and rules.names("x", negated=True) == [] + + +def test_a_tree_without_a_gitignore_ignores_nothing(): + rules = GitIgnore(["src/a.py", "README.md"], lambda f: pytest.fail(f"read {f}")) + assert rules.ignored_by("src/a.py") is None and rules.names("x") == [] + + def test_many_stars_do_not_backtrack(): # A backtracking [^/]* per star takes ~15 s here; the line and the path are repository content. line = "*a" * 9 + "*b"