fix(repo): scan gitignored paths

dokimi/changelog.d/20261026_150000_ndgrigorian_scan-gitignored-paths.md

@@ -0,0 +1,10 @@
+### Fixed
+
+- **`repo scan` no longer fails a path the repository's `.gitignore` ignores** — guidance
+  that names generated or ignored files had each such path reported `fail` as a rename,
+  because its parent directory exists and the file is never tracked. A missing path, directory
+  (with or without its trailing slash) or glob that a tracked `.gitignore` ignores is now
+  `unresolved`, and the reason names the `.gitignore` line. The rules follow gitignore(5), negation and nested files included;
+  `.git/info/exclude` and `core.excludesFile` are not part of the snapshot and are not read.
+  Some paths an earlier scan reported `fail` are now `unresolved`, so the fail count and the L0
+  gate result can differ from a scan of the same repository made before this change.

dokimi/docs/repo-evaluation-contract.md

@@ -1057,7 +1057,26 @@ A path in a code span is tried at the file's directory and each of its
 ancestors. If none has it, the path is `fail` when its parent directory exists
 at one of those bases (a rename or a deleted file), `ok_suffix` when it ends a
 longer path in the tree (a root file naming paths inside a subproject), and
-`unresolved` otherwise (a build output, a path inside a submodule). `ok_suffix`
+`unresolved` otherwise (a build output, a path inside a submodule). A missing
+path that a tracked `.gitignore` ignores is `unresolved` rather than `fail`,
+with the `.gitignore` line in the reason (also for a path that was `unresolved`
+already): guidance that warns about generated files names paths no snapshot
+holds. A directory counts as ignored when git ignores it, or, when no `!` line
+names the directory itself (`!*/` names every directory, so it does not count),
+would ignore every file probed directly inside it: a plain name and each name a
+`!` line re-includes. So `include/*` ignores `include/`, and so does `out/*` with
+`!*/`, while a whitelist of `*`, `!*/`, `!*.py` does not ignore a missing
+`olddir/`, nor `!/setup.py` a missing `setup.py`. A path without a trailing slash counts when it
+would be as a file or as a directory: under that whitelist `*` ignores a missing
+`olddir` read as a file, so a directory needs its trailing slash to be checked
+as a rename. A glob counts when one of the files it is
+probed with would be: its wildcards filled in, and each name an ignoring line
+matches that the glob matches too (`gen.h` for `pkg/g*.h`), when a literal
+starts or ends the glob's last segment: `maintenance/*` or `pkg/*_*` would match
+the name of any rule (`_skbuild`), which says nothing about the glob. The probe
+tries a few names, not every name the glob matches. `.git/info/exclude` and
+`core.excludesFile` belong to a clone, not the tree,
+and are not read. `ok_suffix`
 and `ok_basename` resolve a path but cannot see it renamed, which then reads as
 `unresolved`; guidance the gate must protect names its paths from the repository
 root. A bare name

dokimi/dokimi/repos/l0/core.py

@@ -21,6 +21,7 @@
 from typing import Any
 
 from ..source import RepoTree, SourceError
+from .gitignore import GitIgnore, Rule
 from .markdown import PYTHON_LANGS, SHELL_LANGS, STARLARK_LANGS, Fence, split
 
 # Files whose words are prose, not code: an identifier found only there is not in the code.
@@ -85,6 +86,14 @@ def glob_regex(pat: str) -> re.Pattern[str]:
     return re.compile("".join(out) + r"\Z")
 
 
+_FILLS = ("x", "dokimi-probe")  # what a `*` becomes in a probe of the files a glob names
+
+
+def _fill(pat: str, fill: str) -> str:
+    """``pat`` with its wildcards filled in: ``**/`` as nothing, ``*`` as ``fill``, ``?`` as its first letter."""
+    return re.sub(r"\?", fill[0], re.sub(r"\*+", fill, pat.replace("**/", "")))
+
+
 def is_doc(path: str) -> bool:
     name = posixpath.basename(path)
     if name == "CMakeLists.txt" or name.startswith("requirements"):
@@ -305,11 +314,11 @@ def resolve_path(self, file: str, raw: str) -> tuple[str, str] | None:
         for c in cands:
             parent = posixpath.dirname(c)
             if parent and parent in self.dir_set:
-                return "fail", f"{parent}/ exists but {c} does not"
+                return self._missing([c], rel.endswith("/"), f"{parent}/ exists but {c} does not")
         hits = [x for x in self.by_name.get(posixpath.basename(target), []) if x.endswith("/" + target)]
         if hits:
             return "ok_suffix", f"below a directory the text does not name; {len(hits)} match(es), e.g. {hits[0]}"
-        return "unresolved", "tried " + ", ".join(cands)
+        return self._missing(cands, rel.endswith("/"), "tried " + ", ".join(cands), "unresolved")
 
     def _glob(self, bases: list[str], rel: str) -> tuple[str, str]:
         """A glob is judged like the plain path it generalises (see :meth:`resolve_path`)."""
@@ -327,13 +336,69 @@ def _glob(self, bases: list[str], rel: str) -> tuple[str, str]:
             for b in bases:
                 d = posixpath.normpath(posixpath.join(b, parent))
                 if d in self.dir_set:
-                    return "fail", f"{d}/ exists but nothing in it matches {posixpath.basename(pat)}"
+                    probes = self._probes(posixpath.normpath(posixpath.join(b, pat)))
+                    return self._missing(probes, rel.endswith("/"),
+                                         f"{d}/ exists but nothing in it matches {posixpath.basename(pat)}")
         if parent:
             rx = glob_regex("*/" + pat)
             hit = next((x for x in every if rx.match(x)), None)
             if hit:
                 return "ok_suffix", f"below a directory the text does not name; e.g. {hit}"
-        return "unresolved", f"glob {pat} matches nothing in the tree; a build may create it"
+        probes = [p for b in bases for p in self._probes(posixpath.normpath(posixpath.join(b, pat)))]
+        return self._missing(probes, rel.endswith("/"),
+                             f"glob {pat} matches nothing in the tree; a build may create it", "unresolved")
+
+    def _probes(self, full: str) -> list[str]:
+        """Files the glob ``full`` names, to ask the ``.gitignore`` about: a probe of a few names, not every one.
+
+        Its wildcards filled in, and each name an ignoring rule matches that the glob matches too
+        (``gen.h`` for ``g*.h``); two fillings, so one re-included (``!x.cxx``) leaves the other.
+        Rule names count only for a last segment a literal starts or ends: ``*`` or ``*_*`` matches
+        the name of an unrelated rule (``_skbuild``), which is no evidence the glob is ignored.
+        """
+        folder, name = posixpath.split(full)
+        rx = glob_regex(full)
+        named = name[:1] not in "*?[" or name[-1:] not in "*?]"
+        paths = [posixpath.join(_fill(folder, fill), n) for fill in _FILLS
+                 for n in (_fill(name, fill), *(self.gitignore().names(fill) if named else ()))]
+        return [p for p in dict.fromkeys(paths) if rx.match(p)]
+
+    def _missing(self, paths: list[str], is_dir: bool, why: str, status: str = "fail") -> tuple[str, str]:
+        """``status`` for a path the tree lacks, unless a ``.gitignore`` ignores it: a build output.
+
+        ``paths`` are the files a glob names, or the path at each base it is tried from; any of them
+        being ignored is enough.
+        """
+        rule = next(filter(None, (self._ignored_by(p, is_dir) for p in paths)), None)
+        if rule is None:
+            return status, why
+        return "unresolved", (f"{why}; {rule.source}:{rule.line} (`{rule.pattern}`) ignores it, "
+                              "so it is likely generated rather than tracked")
+
+    def _ignored_by(self, path: str, is_dir: bool) -> Rule | None:
+        """The rule that ignores ``path``, read as a directory too: without a trailing slash it may
+        still name one (``dist`` under ``dist/``).
+
+        A path no ``!`` line names (``!/setup.py``, ``!*.py``; ``!*/`` names every directory, not this
+        one) also counts when every file probed directly inside it would be (``include/*``, or ``out/*``
+        with ``!*/``): a plain name and each name a ``!`` line re-includes, so a whitelist (``*``,
+        ``!*/``, ``!*.py``) does not hide a renamed directory.
+        """
+        ig = self.gitignore()
+        readings = (True,) if is_dir else (False, True)
+        rule = next(filter(None, (ig.ignored_by(path, d) for d in readings)), None)
+        if rule is not None:
+            return rule
+        if any(r and r.name.strip("*") for r in (ig.reincluded_by(path, d) for d in readings)):
+            return None
+        inside = [ig.ignored_by(posixpath.join(path, n)) for n in ("file", *ig.names("x", negated=True))]
+        return inside[0] if all(inside) else None
+
+    def gitignore(self) -> GitIgnore:
+        def load() -> GitIgnore:
+            self.preload([f for f in self.files if posixpath.basename(f) == ".gitignore"])
+            return GitIgnore(self.files, self.text)
+        return self.cached("gitignore", load)
 
     def _every(self) -> list[str]:
         return self.cached("every", lambda: sorted(self.file_set | self.dir_set))

dokimi/dokimi/repos/l0/gitignore.py

@@ -0,0 +1,249 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Which paths a snapshot's ``.gitignore`` files ignore, read from the tree like every L0 fact.
+
+A missing path the repository ignores is a build output the guidance warns about,
+not a rename: L0 reports it ``unresolved``. Only tracked ``.gitignore`` files count;
+``.git/info/exclude`` and ``core.excludesFile`` are per clone and not part of the snapshot.
+
+The rules follow gitignore(5): ``#`` comments and blank lines skipped, ``!``
+re-includes, a trailing ``/`` matches directories only, a pattern with a ``/``
+before its end is anchored to its file's directory and one without matches a
+name at any depth, ``*``, ``?`` and ``[...]`` stay within one segment, ``**`` spans
+segments, a line git's wildmatch aborts on matches nothing, the last matching line wins with deeper files read after shallower
+ones, and nothing under an ignored directory can be re-included.
+"""
+
+from __future__ import annotations
+
+import posixpath
+import re
+from dataclasses import dataclass
+from typing import Callable, Iterable
+
+
+@dataclass(frozen=True)
+class Rule:
+    source: str  # the .gitignore path
+    line: int
+    pattern: str  # the line as written
+    base: str  # the directory the file is in, "" at the root
+    regex: re.Pattern[str]
+    anchored: bool
+    dir_only: bool
+    negated: bool
+    name: str = ""  # the last segment of the pattern, wildcards kept; "" for a line git cannot read
+
+    def matches(self, path: str, is_dir: bool) -> bool:
+        if self.dir_only and not is_dir:
+            return False
+        if self.base:
+            if not path.startswith(self.base + "/"):
+                return False
+            path = path[len(self.base) + 1:]
+        return bool(self.regex.match(path if self.anchored else posixpath.basename(path)))
+
+
+def _segment(pat: str) -> str:
+    """Regex for a glob within path segments: ``*`` and ``?`` never cross ``/``.
+
+    As in CPython's fnmatch, every ``*`` but the last takes the fixed-width run after it
+    in an atomic group, so a line with many stars cannot backtrack exponentially.
+    """
+    runs: list[list[str]] = [[]]  # the fixed-width atoms before, between and after the stars
+    i = 0
+    while i < len(pat):
+        c = pat[i]
+        if c == "[":
+            atom, i = _bracket(pat, i)
+            runs[-1].append(atom)
+            continue
+        if c == "\\":
+            i += 1
+            if i == len(pat):
+                raise _Invalid
+            runs[-1].append(re.escape(pat[i]))
+        elif c == "*":
+            if len(runs) == 1 or runs[-1]:
+                runs.append([])
+        elif c == "?":
+            runs[-1].append("[^/]")
+        else:
+            runs[-1].append(re.escape(c))
+        i += 1
+    head, *rest = ("".join(r) for r in runs)
+    if not rest:
+        return head
+    *middle, tail = rest
+    return head + "".join(f"(?>[^/]*?{m})" for m in middle) + "[^/]*" + tail
+
+
+class _Invalid(Exception):
+    """A line git's wildmatch aborts on (an unclosed ``[``, a trailing ``\\``): it matches nothing."""
+
+
+_CLASSES = {
+    "alnum": "a-zA-Z0-9", "alpha": "a-zA-Z", "blank": r" \t", "cntrl": r"\x00-\x1f\x7f",
+    "digit": "0-9", "graph": "!-~", "lower": "a-z", "print": " -~", "punct": r"!-/:-@\[-`{-~",
+    "space": r" \t\n\r", "upper": "A-Z", "xdigit": "0-9A-Fa-f",
+}
+
+
+def _bracket(pat: str, i: int) -> tuple[str, int]:
+    """The regex for the ``[...]`` at ``pat[i]`` and the index after it, read as git's wildmatch reads it.
+
+    A ``]`` first is a member, ``\\`` escapes, ``[:name:]`` is an ASCII class, a reversed
+    range keeps only its start, and the set never matches ``/``.
+    """
+    i += 1
+    neg = pat[i:i + 1] in ("!", "^")
+    i += neg
+    items: list[str] = []
+    prev = ""
+    while i < len(pat) and (pat[i] != "]" or not items):
+        c = pat[i]
+        if c == "\\" or (c == "-" and prev and pat[i + 1:i + 2] not in ("", "]")):
+            i += 1 + (c == "-" and pat[i + 1] == "\\")
+            if i == len(pat):
+                raise _Invalid
+            if c == "-":
+                items.append(f"{re.escape(prev)}-{re.escape(pat[i])}" if prev <= pat[i] else "")
+                prev = ""
+            else:
+                prev = pat[i]
+                items.append(re.escape(prev))
+        elif c == "[" and pat[i + 1:i + 2] == ":" and (end := pat.find("]", i + 2)) > i + 2 and pat[end - 1] == ":":
+            if pat[i + 2:end - 1] not in _CLASSES:
+                raise _Invalid
+            items.append(_CLASSES[pat[i + 2:end - 1]])
+            prev, i = "", end
+        else:
+            prev = c
+            items.append(re.escape(c))
+        i += 1
+    if i == len(pat):
+        raise _Invalid
+    return "(?!/)[" + ("^" if neg else "") + "".join(items) + "]", i + 1
+
+
+def _split(pat: str) -> list[str]:
+    """``pat`` cut at each ``/`` outside a bracket expression."""
+    parts, start, i = [], 0, 0
+    while i < len(pat):
+        if pat[i] == "[":
+            i = _bracket(pat, i)[1]
+            continue
+        if pat[i] == "/":
+            parts.append(pat[start:i])
+            start = i + 1
+        i += 1 + (pat[i] == "\\")
+    return parts + [pat[start:]]
+
+
+def _regex(pat: str) -> str:
+    """``**/`` leading, ``/**/`` inside and ``/**`` trailing span segments; any other ``**`` is ``*``."""
+    parts = _split(pat)
+    out: list[str] = []
+    for i, part in enumerate(parts):
+        last = i == len(parts) - 1
+        if part == "**":
+            out.append(".*" if last else "(?:[^/]+/)*")
+            continue
+        out.append(_segment(part) + ("" if last else "/"))
+    return "".join(out)
+
+
+_NEVER = re.compile(r"(?!)")
+_MEMBERS = "xad0_-.AZ9"  # tried in turn for a ``[...]``: the first one it matches stands for it
+
+
+def example(pat: str, fill: str) -> str | None:
+    """A name the one-segment ``pat`` matches: each ``*`` as ``fill``, each ``?`` as its first letter,
+    each ``[...]`` as a member; ``None`` when no candidate member fits a class."""
+    out, i = [], 0
+    while i < len(pat):
+        c = pat[i]
+        if c == "[":
+            atom, i = _bracket(pat, i)
+            out.append(next((m for m in fill[0] + _MEMBERS if re.fullmatch(atom, m)), ""))
+            if not out[-1]:
+                return None
+            continue
+        if c == "\\":
+            i += 1
+            out.append(pat[i])
+        else:
+            out.append({"*": fill, "?": fill[0]}.get(c, c))
+        i += 1
+    return "".join(out)
+
+
+def _trim(line: str) -> str:
+    """``line`` without its trailing spaces, as git trims it: a tab stays, and so does a space after a ``\\``."""
+    end, i = len(line), 0
+    while i < len(line):
+        if line[i] != " ":
+            end = len(line)
+            i += line[i] == "\\"  # the escaped character, a space or a second backslash
+        elif end == len(line):
+            end = i
+        i += 1
+    return line[:end]
+
+
+def parse(source: str, text: str) -> list[Rule]:
+    base = posixpath.dirname(source)
+    rules = []
+    # git splits at "\n" only and drops a "\r" before it, and skips a byte-order mark
+    for n, raw in enumerate(text.removeprefix("\ufeff").split("\n"), 1):
+        line = _trim(raw.removesuffix("\r"))
+        if not line or line.startswith("#"):
+            continue
+        negated = line.startswith("!")
+        pat = line[1:] if negated else line
+        if pat.startswith(("\\#", "\\!")):
+            pat = pat[1:]
+        dir_only = pat.endswith("/")
+        pat = pat.rstrip("/")
+        if not pat:
+            continue
+        anchored = "/" in pat
+        try:
+            regex, name = re.compile(_regex(pat.lstrip("/")) + r"\Z"), _split(pat.lstrip("/"))[-1]
+        except _Invalid:
+            regex, name = _NEVER, ""
+        rules.append(Rule(source, n, line, base, regex, anchored, dir_only, negated, name))
+    return rules
+
+
+class GitIgnore:
+    """The ignore rules of every ``.gitignore`` in a tree."""
+
+    def __init__(self, files: Iterable[str], read: Callable[[str], str]) -> None:
+        sources = sorted((f for f in files if posixpath.basename(f) == ".gitignore"),
+                         key=lambda f: (f.count("/"), f))
+        self.rules = [r for f in sources for r in parse(f, read(f))]
+
+    def _last(self, path: str, is_dir: bool) -> Rule | None:
+        return next((r for r in reversed(self.rules) if r.matches(path, is_dir)), None)
+
+    def names(self, fill: str, negated: bool = False) -> list[str]:
+        """A name the last segment of each ignoring rule matches (``gen_a.h`` for ``gen_[a-c].h``),
+        or with ``negated`` of each ``!`` rule."""
+        found = (example(r.name, fill) for r in self.rules if r.name and r.negated == negated)
+        return list(dict.fromkeys(n for n in found if n))
+
+    def reincluded_by(self, path: str, is_dir: bool = False) -> Rule | None:
+        """The ``!`` line that matches ``path`` itself last, or ``None`` when the last match ignores it or none does."""
+        rule = self._last(posixpath.normpath(path).strip("/"), is_dir)
+        return rule if rule is not None and rule.negated else None
+
+    def ignored_by(self, path: str, is_dir: bool = False) -> Rule | None:
+        """The rule that ignores ``path`` or one of its directories, or ``None`` when git would not."""
+        parts = posixpath.normpath(path).strip("/").split("/")
+        for i in range(1, len(parts) + 1):
+            rule = self._last("/".join(parts[:i]), is_dir or i < len(parts))
+            if rule is not None and not rule.negated:
+                return rule
+        return None