diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index 3b3ebc92..d4344d0a 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -1094,6 +1094,7 @@ intended to be matched in queries or to show additional information. .. _Git: http://git-scm.com/ .. _Svn: http://subversion.apache.org/ +.. _rsync: https://rsync.samba.org/ Most SCMs support the ``sslVerify`` attribute. This is a boolean that controls whether to verify the SSL certificate when fetching. If unset, it defaults to @@ -1296,6 +1297,10 @@ import +----------------------------+-------------------------------------------------------------------------------+ | ``recipeRelative``\* | Whether ``url`` is relative to recipe or project root. (optional) | +----------------------------+-------------------------------------------------------------------------------+ + | ``filter`` | Ordered list of rsync-like include/exclude rules. (optional) | + +----------------------------+-------------------------------------------------------------------------------+ + | ``pruneEmptyDirs``\* | Do not import empty directories. Default: ``False``. (optional) | + +----------------------------+-------------------------------------------------------------------------------+ The ``import`` SCM copies the directory specified in ``url`` to the workspace. By default, the destination is always overwritten and obsolete @@ -1320,6 +1325,63 @@ import itself if ``recipeRelative`` is set to ``True``. This is recommended especially for recipes that are included as layers into other projects. + The ``filter`` attribute restricts which entries of the ``url`` directory + are imported. This is useful if a directory holds files for different + packages, e.g. ``defconfig`` files for different boards. Only the selected + files end up in the workspace. A package is thus not rebuilt if other files + in the directory are changed. Example:: + + checkoutSCM: + scm: import + url: configs + filter: + - "- *.orig" + - "+ /board-a_defconfig" + - "+ /patches/" + - "+ /patches/board-a/***" + - "- *" + + The filter rules work like the include/exclude filter rules of `rsync`_. + Each rule is either an include rule (``+ PATTERN`` or ``include PATTERN``) + or an exclude rule (``- PATTERN`` or ``exclude PATTERN``). The rule name + and the pattern are separated by exactly one space. Like in rsync filter + files, a rule cannot contain line breaks. The rules are applied as + follows: + + * Each file, directory and symlink is checked against the rules in the + given order. The first matching rule decides whether the entry is + imported or not. Entries that do not match any rule are imported. + * Excluded directories are not traversed. If something below a directory + shall be imported, the directory itself must be included too. In the + example above, the ``/patches/`` rule is required. Otherwise the + final ``- *`` rule would exclude the ``patches`` directory and the + ``board-a`` patches would never be considered. A common idiom to + include certain files at any depth is ``+ */``, ``+ *.patch``, + ``- *`` together with ``pruneEmptyDirs``. + * If the rules exclude everything, the checkout fails. + + The patterns are matched against the path of each entry relative to + ``url``, using ``/`` as path separator: + + * A pattern that starts with ``/`` is anchored at the ``url`` directory. + Otherwise it may match at any depth. A pattern without ``/`` (not + counting a trailing one) or ``**`` is matched against the final path + component only. Otherwise it is matched against the trailing path + components, e.g. ``a/b`` matches ``a/b`` and ``x/a/b``. + * A pattern that ends with ``/`` matches only directories. + * ``*`` matches anything except ``/``. ``**`` matches anything, including + ``/``. ``?`` matches any single character except ``/``. ``[...]`` + matches one character in the set. A set that starts with ``!`` or ``^`` + is negated. Use ``\`` to match a wildcard character literally. + * A trailing ``dir/***`` matches the directory itself and everything + inside it. + + Directories that are empty after applying the filter rules are still + created in the workspace. Set ``pruneEmptyDirs`` to ``True`` to skip all + empty directories (like the ``--prune-empty-dirs`` option of rsync). This + applies to empty directories in the source too, even if no filter rules + are given. + svn The `Svn`_ SCM, like git, requires the ``url`` attribute too. If you specify a numeric ``revision`` Bob considers the SCM as deterministic. diff --git a/pym/bob/scm/imp.py b/pym/bob/scm/imp.py index bde5edf3..f8bb552b 100644 --- a/pym/bob/scm/imp.py +++ b/pym/bob/scm/imp.py @@ -3,31 +3,214 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from ..errors import BuildError +from ..errors import BuildError, ParseError from ..stringparser import IfExpression -from ..tty import stepAction, INFO, TRACE +from ..tty import stepAction, INFO, TRACE, WARNING from ..utils import asHexStr, hashDirectory, emptyDirectory, tarfileOpen from .scm import Scm, ScmAudit import base64 +import functools import io import os, os.path +import posixpath +import re import schema import shutil import stat +import tempfile -def copyTree(src, dst, invoker): +def _translateGlob(pattern): + r"""Translate rsync wildcard pattern into regular expression. + + The following wildcards are supported: + + * ``*`` matches any path component, but it stops at slashes. + * ``**`` matches anything, including slashes. + * ``?`` matches any character except a slash. + * ``[...]`` matches a character in the set. A leading ``!`` or ``^`` + negates the set. Never matches a slash. + * ``\`` escapes the next character. + """ + i, n = 0, len(pattern) + res = [] + while i < n: + c = pattern[i] + i += 1 + if c == '*': + if i < n and pattern[i] == '*': + while i < n and pattern[i] == '*': + i += 1 + res.append('.*') + else: + res.append('[^/]*') + elif c == '?': + res.append('[^/]') + elif c == '[': + j = i + if j < n and pattern[j] in '!^': j += 1 + if j < n and pattern[j] == ']': j += 1 + while j < n and pattern[j] != ']': j += 1 + if j >= n: + res.append(re.escape(c)) + else: + chars = pattern[i:j] + i = j+1 + negate = chars[:1] in ('!', '^') + if negate: chars = chars[1:] + chars = "".join(('\\' + x) if x in '\\[]^&~|' else x for x in chars) + res.append('(?!/)[' + ('^' if negate else '') + chars + ']') + elif c == '\\' and i < n: + res.append(re.escape(pattern[i])) + i += 1 + else: + res.append(re.escape(c)) + return "".join(res) + +class FilterRule: + """A single rsync-like filter rule. + + Rules are either given in their short ("+ PATTERN", "- PATTERN") or long + form ("include PATTERN", "exclude PATTERN"). The pattern is matched like + rsync does: + + * A leading ``/`` anchors the pattern at the root of the imported + directory. Otherwise the pattern may match at any depth. + * A trailing ``/`` matches only directories. + * A pattern without ``/`` or ``**`` is matched against the final path + component only. + * A trailing ``dir/***`` matches the directory and everything inside it. + """ + + # Like in rsync filter files, rules cannot span multiple lines. + RULE_RE = re.compile(r'^(\+|-|include|exclude) ([^\n]+)\Z') + + def __init__(self, rule): + m = self.RULE_RE.match(rule) + if m is None: + raise ParseError("Invalid import filter rule: '{}'".format(rule), + help="Rules must be of the form '+ PATTERN', '- PATTERN', " + "'include PATTERN' or 'exclude PATTERN'.") + self.include = m.group(1) in ('+', 'include') + pattern = m.group(2) + + self.dirOnly = pattern.endswith('/') and pattern != '/' + if self.dirOnly: + pattern = pattern[:-1] + + try: + # Trailing "/***" matches the directory itself and everything below. + self.dirItself = None + if pattern.endswith('/***'): + self.dirItself = self.__compile(pattern[:-4]) + self.regex = self.__compile(pattern) + except re.error as e: + raise ParseError("Invalid import filter rule '{}': {}".format(rule, str(e))) + + @staticmethod + def __compile(pattern): + """Compile pattern to (regex, prependSlash) tuple.""" + if pattern.startswith('/'): + # Anchored at root. + return (re.compile(_translateGlob(pattern[1:]), re.DOTALL), False) + elif pattern.startswith('**'): + # Like rsync, let "**/" also match at the start of the path by + # matching against the path with a leading slash. + return (re.compile(_translateGlob(pattern), re.DOTALL), True) + else: + # Match against the trailing path components. + return (re.compile('(?:.*/)?' + _translateGlob(pattern), re.DOTALL), False) + + def matches(self, path, isDir): + if self.dirOnly and not isDir: + return False + if isDir and self.dirItself is not None and self.__match(self.dirItself, path): + return True + return self.__match(self.regex, path) + + @staticmethod + def __match(regex, path): + regex, prependSlash = regex + if prependSlash: + path = '/' + path + return regex.fullmatch(path) is not None + +@functools.lru_cache(maxsize=None) +def parseFilterRule(rule): + return FilterRule(rule) + +def selectTree(src, rules, pruneEmptyDirs): + """Compute the set of imported paths. + + Returns the set of paths (relative to src, using '/' as separator) that + are imported. Returns None if no filtering is requested, i.e. everything + is imported. + + The rules are evaluated in order for each entry. The first matching rule + determines whether the entry is included or excluded. Entries that do not + match any rule are included. Like rsync, excluded directories are not + traversed. Optionally, directories that do not contain any imported + entries are skipped too. + """ + + if not rules and not pruneEmptyDirs: + return None + + rules = [ parseFilterRule(r) for r in rules ] + selected = set() + + def isIncluded(path, isDir): + for r in rules: + if r.matches(path, isDir): + return r.include + return True + + def scan(srcDir, relDir): + found = False + with os.scandir(srcDir) as it: + for entry in it: + path = posixpath.join(relDir, entry.name) + isDir = entry.is_dir(follow_symlinks=False) + if not isIncluded(path, isDir): + continue + if isDir: + if scan(entry.path, path) or not pruneEmptyDirs: + selected.add(path) + found = True + else: + selected.add(path) + found = True + return found + + try: + scan(src, "") + except OSError as e: + raise BuildError("Error gathering files: {}".format(str(e))) + + if rules and not selected: + raise BuildError("Cannot import '{}': filter rules exclude everything!".format(src)) + + return selected + +def copyTree(src, dst, selection=None, relDir=""): """Recursively copy directory tree. The src and dst directories must already exist. The items in the source directory are copied to the destination only if it does not exist yet or if it is newer than the destination. The copy operation is aborted if the source and destination file types differ (file vs. directory vs. symlink). + + If a selection is given (see selectTree()), only the paths in the set are + copied. """ changed = False try: todo = [] for name in os.listdir(src): + relName = posixpath.join(relDir, name) + if (selection is not None) and (relName not in selection): + continue + srcName = os.path.join(src, name) dstName = os.path.join(dst, name) @@ -39,21 +222,21 @@ def copyTree(src, dst, invoker): if dstStat is not None: if stat.S_IFMT(srcStat.st_mode ^ dstStat.st_mode) != 0: - invoker.fail("Copy failed: destination has different type:", srcName) + raise BuildError("Copy failed: destination has different type: " + srcName) - todo.append((srcName, srcStat, dstName, dstStat)) + todo.append((srcName, srcStat, dstName, dstStat, relName)) # First handle everything *except* symlinks. On MSYS the destination # must exist when creating the symlink, probably because the target # type must be determined on creation. - for srcName, srcStat, dstName, dstStat in todo: + for srcName, srcStat, dstName, dstStat, relName in todo: if stat.S_ISLNK(srcStat.st_mode): pass elif stat.S_ISDIR(srcStat.st_mode): if dstStat is None: os.mkdir(dstName) changed = True - if copyTree(srcName, dstName, invoker): + if copyTree(srcName, dstName, selection, relName): shutil.copystat(srcName, dstName) else: if (dstStat is not None) and (srcStat.st_mtime_ns <= dstStat.st_mtime_ns): @@ -62,7 +245,7 @@ def copyTree(src, dst, invoker): changed = True # Now handle all symlinks. - for srcName, srcStat, dstName, dstStat in todo: + for srcName, srcStat, dstName, dstStat, relName in todo: if stat.S_ISLNK(srcStat.st_mode): linkTo = os.readlink(srcName) if dstStat is not None: @@ -73,14 +256,15 @@ def copyTree(src, dst, invoker): os.symlink(linkTo, dstName) changed = True except OSError as e: - invoker.fail("Copy failed", str(e)) + raise BuildError("Copy failed: " + str(e)) return changed -def packTree(src): +def packTree(src, rules=[], pruneEmptyDirs=False): if not os.path.isdir(src): raise BuildError("Cannot import '{}': not a directory!".format(src)) + selection = selectTree(src, rules, pruneEmptyDirs) try: f = io.BytesIO() with tarfileOpen(fileobj=f, mode="w:xz") as tar: @@ -90,6 +274,10 @@ def packTree(src): # and then the symlinks are added. symlinks = [] def filterSymlinks(ti): + if selection is not None: + name = posixpath.normpath(ti.name) + if (name != ".") and (name not in selection): + return None if ti.issym(): symlinks.append(ti) return None @@ -117,12 +305,15 @@ class ImportScm(Scm): schema.Optional('dir') : str, schema.Optional('prune') : bool, schema.Optional('recipeRelative') : bool, + schema.Optional('pruneEmptyDirs') : bool, } __SCHEMA = { 'scm' : 'import', 'url' : str, schema.Optional('if') : schema.Or(str, IfExpression), + schema.Optional('filter') : [schema.Regex(FilterRule.RULE_RE.pattern, + error="Invalid import filter rule '{}'. Expected '+ PATTERN', '- PATTERN', 'include PATTERN' or 'exclude PATTERN'.")], } SCHEMA = schema.Schema({**__SCHEMA, **DEFAULTS}) @@ -130,6 +321,8 @@ class ImportScm(Scm): DEFAULT_VALUES = { "dir" : ".", "recipeRelative" : False, + "filter" : [], + "pruneEmptyDirs" : False, } def __init__(self, spec, overrides=[], pruneDefault=None, fixDigestBug=False, projectRoot=""): @@ -141,6 +334,10 @@ def __init__(self, spec, overrides=[], pruneDefault=None, fixDigestBug=False, pr self.__projectRoot = spec.get("__projectRoot", projectRoot) self.__fixDigestBug = fixDigestBug self.__recipeRelative = spec.get("recipeRelative", False) + self.__filter = spec.get("filter", []) + self.__pruneEmptyDirs = spec.get("pruneEmptyDirs", False) + # Validate rules early + for r in self.__filter: parseFilterRule(r) def _getSrcDir(self): rootDir = os.path.dirname(self._getRecipe()) if self.__recipeRelative else self.__projectRoot @@ -154,9 +351,11 @@ def getProperties(self, isJenkins, pretty=False): 'dir' : self.__dir, 'prune' : self.__prune, 'recipeRelative' : self.__recipeRelative, + 'filter' : self.__filter, + 'pruneEmptyDirs' : self.__pruneEmptyDirs, }) if isJenkins: - ret['__data'] = packTree(self._getSrcDir()) + ret['__data'] = packTree(self._getSrcDir(), self.__filter, self.__pruneEmptyDirs) elif not pretty: ret['__projectRoot'] = self.__projectRoot @@ -174,15 +373,26 @@ async def invoke(self, invoker, workspaceCreated): src = self._getSrcDir() if not os.path.isdir(src): invoker.fail("Cannot import '{}': not a directory!".format(src)) - copyTree(src, dest, invoker) + try: + copyTree(src, dest, selectTree(src, self.__filter, self.__pruneEmptyDirs)) + except BuildError as e: + invoker.fail(e.slogan) else: unpackTree(self.__data, dest) def asDigestScript(self): if self.__fixDigestBug: - return self.__url + " " + self.__dir + ret = self.__url + " " + self.__dir else: - return self.__url + ret = self.__url + # Only add new options if used to keep the variant-id of existing + # recipes stable. Each one is put on a separate line. Rules cannot + # contain newlines so this is unambiguous. + if self.__pruneEmptyDirs: + ret += "\npruneEmptyDirs" + for rule in self.__filter: + ret += "\nfilter " + rule + return ret def getDirectory(self): return self.__dir @@ -198,13 +408,29 @@ def hasLiveBuildId(self): async def predictLiveBuildId(self, step): with stepAction(step, "HASH", self.__url, (INFO, TRACE)) as a: - return hashDirectory(self.__url) + src = self._getSrcDir() + if not self.__filter and not self.__pruneEmptyDirs: + return hashDirectory(src) + + # The workspace will only hold the selected files. Make a + # temporary copy of them to calculate the expected hash. + try: + selection = selectTree(src, self.__filter, self.__pruneEmptyDirs) + with tempfile.TemporaryDirectory() as tmp: + copyTree(src, tmp, selection) + return hashDirectory(tmp) + except BuildError as e: + a.fail(e.slogan, WARNING) + return None def calcLiveBuildId(self, workspacePath): return hashDirectory(os.path.join(workspacePath, self.__dir)) def getAuditSpec(self): - return ("import", self.__dir, {"url" : self.__url}) + extra = {"url" : self.__url} + if self.__filter: extra["filter"] = self.__filter + if self.__pruneEmptyDirs: extra["pruneEmptyDirs"] = True + return ("import", self.__dir, extra) class ImportAudit(ScmAudit): @@ -217,20 +443,26 @@ class ImportAudit(ScmAudit): 'value' : str }, 'url' : str, + schema.Optional('filter') : [str], + schema.Optional('pruneEmptyDirs') : bool, }) async def _scanDir(self, workspace, dir, extra): self.__dir = dir self.__hash = asHexStr(hashDirectory(os.path.join(workspace, dir))) self.__url = extra.get("url") + self.__filter = extra.get("filter", []) + self.__pruneEmptyDirs = extra.get("pruneEmptyDirs", False) def _load(self, data): self.__dir = data["dir"] self.__hash = data["digest"]["value"] self.__url = data["url"] + self.__filter = data.get("filter", []) + self.__pruneEmptyDirs = data.get("pruneEmptyDirs", False) def dump(self): - return { + ret = { "type" : "import", "dir" : self.__dir, "digest" : { @@ -239,3 +471,6 @@ def dump(self): }, "url" : self.__url, } + if self.__filter: ret["filter"] = self.__filter + if self.__pruneEmptyDirs: ret["pruneEmptyDirs"] = True + return ret diff --git a/test/black-box/import-scm-filter/config.yaml b/test/black-box/import-scm-filter/config.yaml new file mode 100644 index 00000000..5c45369e --- /dev/null +++ b/test/black-box/import-scm-filter/config.yaml @@ -0,0 +1 @@ +bobMinimumVersion: "1.3" diff --git a/test/black-box/import-scm-filter/root.yaml b/test/black-box/import-scm-filter/root.yaml new file mode 100644 index 00000000..0c6161bc --- /dev/null +++ b/test/black-box/import-scm-filter/root.yaml @@ -0,0 +1,23 @@ +root: True +multiPackage: + a: + checkoutSCM: + scm: import + url: src + filter: + - "+ /a_*" + - "- *" + b: + checkoutSCM: + scm: import + url: src + pruneEmptyDirs: True + filter: + - "exclude *.orig" + - "include /b_*" + - "include /patches/***" + - "exclude *" +buildScript: | + cp -a "$1/"* . + date +%s%N > stamp +packageScript: cp -a "$1/"* . diff --git a/test/black-box/import-scm-filter/run.sh b/test/black-box/import-scm-filter/run.sh new file mode 100755 index 00000000..61cda97b --- /dev/null +++ b/test/black-box/import-scm-filter/run.sh @@ -0,0 +1,54 @@ +#!/bin/bash -e +# +# Test filter rules of the "import" SCM. Packages that import +# different files from the same directory must not be rebuilt if an unrelated +# file changes. +# +source "$(dirname "$0")/../../test-lib.sh" "../../.." + +cleanup + +# Copy recipes and create initial sources +prj="$(mktemp -d)" +trap 'rm -rf "$prj"' EXIT +mkdir -p "$prj/recipes" "$prj/src/patches" "$prj/src/patches/empty" +cp config.yaml "$prj/" +cp root.yaml "$prj/recipes" +echo "a1" > "$prj/src/a_defconfig" +echo "b1" > "$prj/src/b_defconfig" +echo "p1" > "$prj/src/patches/0001.patch" +echo "o1" > "$prj/src/patches/0001.patch.orig" +echo "x1" > "$prj/src/unrelated.txt" + +run_bob -C "$prj" dev root-a root-b +distA="$prj/$(run_bob -C "$prj" query-path -f '{dist}' root-a)" +distB="$prj/$(run_bob -C "$prj" query-path -f '{dist}' root-b)" + +# Only the selected files are imported +expect_exist "$distA/a_defconfig" +expect_not_exist "$distA/b_defconfig" "$distA/patches" "$distA/unrelated.txt" +expect_exist "$distB/b_defconfig" "$distB/patches/0001.patch" +expect_not_exist "$distB/a_defconfig" "$distB/patches/0001.patch.orig" "$distB/unrelated.txt" \ + "$distB/patches/empty" + +stampA="$(cat "$distA/stamp")" +stampB="$(cat "$distB/stamp")" + +# Changing a file that is imported by "b" only rebuilds "b" +echo "b2" > "$prj/src/b_defconfig" +run_bob -C "$prj" dev root-a root-b +expect_equal "$stampA" "$(cat "$distA/stamp")" +expect_not_equal "$stampB" "$(cat "$distB/stamp")" +expect_equal "b2" "$(cat "$distB/b_defconfig")" +stampB="$(cat "$distB/stamp")" + +# Changing files that are not imported does not rebuild anything +echo "x2" > "$prj/src/unrelated.txt" +echo "o2" > "$prj/src/patches/0001.patch.orig" +run_bob -C "$prj" dev root-a root-b +expect_equal "$stampA" "$(cat "$distA/stamp")" +expect_equal "$stampB" "$(cat "$distB/stamp")" + +# Filter rules that exclude everything are an error +rm "$prj/src/a_defconfig" +expect_fail run_bob -C "$prj" dev root-a diff --git a/test/unit/test_scm_import.py b/test/unit/test_scm_import.py index c6e40d1d..f29b69e6 100644 --- a/test/unit/test_scm_import.py +++ b/test/unit/test_scm_import.py @@ -7,12 +7,13 @@ from unittest.mock import MagicMock, patch import asyncio import os +import schema import tempfile -from bob.errors import BuildError +from bob.errors import BuildError, ParseError from bob.scm.imp import ImportScm, ImportAudit from bob.invoker import Invoker, InvocationError -from bob.utils import hashDirectory, runInEventLoop +from bob.utils import asHexStr, hashDirectory, runInEventLoop class DummyPackage: def getName(self): @@ -177,3 +178,219 @@ def testOutOfTree(self): with tempfile.TemporaryDirectory() as workspace: self.invokeScm(workspace, s) self.assertEqual(digest, hashDirectory(workspace)) + + +class TestImportScmFilter(TestCase): + """Test rsync-like filter rules of import SCM""" + + @classmethod + def setUpClass(cls): + cls.__repodir = tempfile.TemporaryDirectory() + cls.url = os.path.abspath(cls.__repodir.name) + + def mkfile(*path): + os.makedirs(os.path.join(cls.url, *path[:-1]), exist_ok=True) + with open(os.path.join(cls.url, *path), "w") as f: + f.write("/".join(path)) + + mkfile("a_defconfig") + mkfile("b_defconfig") + mkfile("readme.txt") + os.symlink("a_defconfig", os.path.join(cls.url, "link")) + mkfile("patches", "a", "0001.patch") + mkfile("patches", "a", "0001.patch.orig") + mkfile("patches", "b", "0001.patch") + mkfile("x", "a_defconfig") + os.mkdir(os.path.join(cls.url, "empty")) + + @classmethod + def tearDownClass(cls): + cls.__repodir.cleanup() + + def createImportScm(self, spec = {}, **kwargs): + s = { + 'scm' : 'import', + 'url' : self.url, + 'recipe' : "foo.yaml#0", + '__source' : "Recipe foo", + } + s.update(spec) + return ImportScm(s, **kwargs) + + def invokeScm(self, workspace, scm): + spec = MagicMock(workspaceWorkspacePath=workspace, envWhiteList=set()) + invoker = Invoker(spec, False, True, True, True, True, False) + runInEventLoop(scm.invoke(invoker, False)) + + @staticmethod + def listTree(root): + ret = set() + for dirpath, dirnames, filenames in os.walk(root): + rel = os.path.relpath(dirpath, root) + for n in dirnames + filenames: + ret.add(os.path.normpath(os.path.join(rel, n)).replace(os.sep, "/")) + return ret + + def checkout(self, rules, pruneEmptyDirs=False): + """Checkout locally and via Jenkins properties. Must be identical.""" + spec = { "filter" : rules } + if pruneEmptyDirs: spec["pruneEmptyDirs"] = True + s = self.createImportScm(spec) + with tempfile.TemporaryDirectory() as workspace: + self.invokeScm(workspace, s) + local = self.listTree(workspace) + digest = hashDirectory(workspace) + self.assertEqual(digest, s.calcLiveBuildId(workspace)) + self.assertEqual(digest, runInEventLoop(s.predictLiveBuildId(DummyStep()))) + with tempfile.TemporaryDirectory() as workspace: + self.invokeScm(workspace, ImportScm(s.getProperties(True))) + self.assertEqual(local, self.listTree(workspace)) + self.assertEqual(digest, hashDirectory(workspace)) + return local + + ALL = { "a_defconfig", "b_defconfig", "readme.txt", "link", "patches", + "patches/a", "patches/a/0001.patch", "patches/a/0001.patch.orig", + "patches/b", "patches/b/0001.patch", "x", "x/a_defconfig", "empty" } + + def testNoFilter(self): + """Without filter, everything is imported""" + self.assertEqual(self.checkout([]), self.ALL) + + def testSingleFile(self): + """Single files can be imported. Short and long form are equivalent.""" + self.assertEqual(self.checkout(["+ /a_defconfig", "- *"]), {"a_defconfig"}) + self.assertEqual(self.checkout(["include /a_defconfig", "exclude *"]), {"a_defconfig"}) + + def testDefaultInclude(self): + """Entries that match no rule are included""" + self.assertEqual(self.checkout(["- *.orig"]), + self.ALL - {"patches/a/0001.patch.orig"}) + + def testFirstMatchWins(self): + """The first matching rule decides""" + # Directory "x" is excluded by the last rule and never traversed + self.assertEqual(self.checkout(["- b_defconfig", "+ *_defconfig", "- *"]), + {"a_defconfig"}) + self.assertEqual(self.checkout(["- b_defconfig", "+ *_defconfig", "+ */", "- *"]), + {"a_defconfig", "x", "x/a_defconfig", "patches", "patches/a", + "patches/b", "empty"}) + + def testAnchoring(self): + """Patterns are anchored by a leading slash""" + self.assertEqual(self.checkout(["- a_defconfig"]), + self.ALL - {"a_defconfig", "x/a_defconfig"}) + self.assertEqual(self.checkout(["- /a_defconfig"]), + self.ALL - {"a_defconfig"}) + # Unanchored patterns with slashes match trailing path components + self.assertEqual(self.checkout(["- a/0001.patch"]), + self.ALL - {"patches/a/0001.patch"}) + + def testExcludedDirNotTraversed(self): + """Like rsync, excluded directories are not traversed""" + self.assertEqual(self.checkout(["+ /patches/", "+ /patches/a/***", "- *"]), + {"patches", "patches/a", "patches/a/0001.patch", + "patches/a/0001.patch.orig"}) + self.assertEqual(self.checkout(["+ /patches/", "+ /patches/a/", "+ *.patch", "- *"]), + {"patches", "patches/a", "patches/a/0001.patch"}) + + def testDirOnly(self): + """A trailing slash matches only directories""" + self.assertEqual(self.checkout(["- link/", "- empty/"]), self.ALL - {"empty"}) + + def testWildcards(self): + """Test '*', '**', '?' and character sets""" + self.assertEqual(self.checkout(["- /*/a/"]), + self.ALL - {"patches/a", "patches/a/0001.patch", + "patches/a/0001.patch.orig"}) + self.assertEqual(self.checkout(["- patches/**"]), + {"a_defconfig", "b_defconfig", "readme.txt", "link", + "patches", "x", "x/a_defconfig", "empty"}) + self.assertEqual(self.checkout(["- **/a_defconfig"]), + self.ALL - {"a_defconfig", "x/a_defconfig"}) + self.assertEqual(self.checkout(["- ?_defconfig"]), + self.ALL - {"a_defconfig", "b_defconfig", "x/a_defconfig"}) + self.assertEqual(self.checkout(["- [!a]_defconfig"]), + self.ALL - {"b_defconfig"}) + + def testIncludePattern(self): + """Common rsync idiom to include files by pattern""" + patches = {"patches/a/0001.patch", "patches/b/0001.patch"} + self.assertEqual(self.checkout(["+ */", "+ *.patch", "- *"]), + patches | {"patches", "patches/a", "patches/b", "x", "empty"}) + self.assertEqual(self.checkout(["+ */", "+ *.patch", "- *"], True), + patches | {"patches", "patches/a", "patches/b"}) + + def testPruneEmptyDirs(self): + """Empty directories are pruned on request, even without filter""" + self.assertEqual(self.checkout([], True), self.ALL - {"empty"}) + + def testNothingImported(self): + """Filter rules that exclude everything are an error""" + s = self.createImportScm({"filter" : ["+ /c_defconfig", "- *"]}) + with tempfile.TemporaryDirectory() as workspace: + with self.assertRaises(InvocationError): + self.invokeScm(workspace, s) + with self.assertRaises(BuildError): + s.getProperties(True) + self.assertIsNone(runInEventLoop(s.predictLiveBuildId(DummyStep()))) + + def testInvalidRules(self): + """Malformed rules are rejected""" + for r in ["+foo", "+ ", "x foo", "includefoo", "- [z-a]", "+ a\n", "+ a\n- b"]: + with self.assertRaises((schema.SchemaError, ParseError)): + self.createImportScm(ImportScm.SCHEMA.validate( + {"scm" : "import", "url" : self.url, "filter" : [r]})) + + def testFilterKeepsWorkspaceFiles(self): + """Filter does not touch files that are already in the workspace""" + s = self.createImportScm({"filter" : ["+ /a_defconfig", "- *"]}) + with tempfile.TemporaryDirectory() as workspace: + with open(os.path.join(workspace, "canary"), "w") as f: + pass + self.invokeScm(workspace, s) + self.assertEqual(self.listTree(workspace), {"a_defconfig", "canary"}) + + def testDigest(self): + """Filters are part of the digest but only if used""" + def digest(spec): + return self.createImportScm(spec, fixDigestBug=True).asDigestScript() + + plain = digest({}) + self.assertEqual(plain, self.url + " .") + self.assertEqual(plain, digest({"filter" : [], "pruneEmptyDirs" : False})) + + # The format must stay stable. Otherwise variant-ids would change! + rules = digest({"filter" : ["+ a", "- *"]}) + self.assertEqual(rules, self.url + " .\nfilter + a\nfilter - *") + self.assertEqual(digest({"filter" : ["+ a"], "pruneEmptyDirs" : True}), + self.url + " .\npruneEmptyDirs\nfilter + a") + self.assertEqual(self.createImportScm({"filter" : ["+ a"]}).asDigestScript(), + self.url + "\nfilter + a") + self.assertNotEqual(rules, digest({"filter" : ["- *", "+ a"]})) + self.assertNotEqual(rules, digest({"filter" : ["+ a", "- *"], "pruneEmptyDirs" : True})) + self.assertNotEqual(plain, digest({"pruneEmptyDirs" : True})) + + def testProperties(self): + """Options are in properties but hidden from pretty print if unset""" + p = self.createImportScm().getProperties(False, True) + self.assertNotIn("filter", p) + self.assertNotIn("pruneEmptyDirs", p) + + s = self.createImportScm({"filter" : ["+ a", "- *"], "pruneEmptyDirs" : True}) + p = s.getProperties(False, True) + self.assertEqual(p["filter"], ["+ a", "- *"]) + self.assertEqual(p["pruneEmptyDirs"], True) + + def testAudit(self): + """Filters are recorded in the audit trail""" + s = self.createImportScm({"filter" : ["+ /a_defconfig", "- *"]}) + with tempfile.TemporaryDirectory() as workspace: + self.invokeScm(workspace, s) + _, auditDir, extra = s.getAuditSpec() + audit = runInEventLoop(ImportAudit.fromDir(workspace, auditDir, extra)) + d = audit.dump() + self.assertEqual(d["digest"]["value"], asHexStr(hashDirectory(workspace))) + self.assertEqual(d["filter"], ["+ /a_defconfig", "- *"]) + self.assertNotIn("pruneEmptyDirs", d) + ImportAudit.SCHEMA.validate(d) + self.assertEqual(d, ImportAudit.fromData(d).dump())