From 2695ece9f877ccdd81af72067c14b9aa7858b66a Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Mon, 9 Nov 2026 15:09:22 +0100 Subject: [PATCH 01/10] utils: move dumpFlat from cmds/helpers --- pym/bob/cmds/helpers.py | 17 ++--------------- pym/bob/utils.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/pym/bob/cmds/helpers.py b/pym/bob/cmds/helpers.py index abe19fc2..f3ddfab8 100644 --- a/pym/bob/cmds/helpers.py +++ b/pym/bob/cmds/helpers.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later +from ..utils import dumpFlat + def processDefines(defs): """Convert a list of 'KEY=VALUE' strings into a dict""" defines = {} @@ -40,18 +42,3 @@ def dumpJson(doc, indent): import json return json.dumps(doc, indent=indent, sort_keys=True) -def dumpFlat(doc, prefix=""): - ret = [] - if isinstance(doc, dict): - for k,v in sorted(doc.items()): - ret.extend(dumpFlat(v, prefix+"."+k if prefix else k)) - elif isinstance(doc, list): - i = 0 - for v in doc: - ret.extend(dumpFlat(v, "{}[{}]".format(prefix, i))) - i += 1 - else: - doc = str(doc).replace('\n', '\\n') - ret = [ "{}={}".format(prefix, doc) ] - - return ret diff --git a/pym/bob/utils.py b/pym/bob/utils.py index fc266e84..49209e89 100644 --- a/pym/bob/utils.py +++ b/pym/bob/utils.py @@ -965,3 +965,20 @@ def tarfileOpen(*args, **kwargs): # Set a sensible extraction filter, used by Python 3.12 and later... ret.extraction_filter = _tarExtractFilter return ret + + +def dumpFlat(doc, prefix=""): + ret = [] + if isinstance(doc, dict): + for k,v in sorted(doc.items()): + ret.extend(dumpFlat(v, prefix+"."+k if prefix else k)) + elif isinstance(doc, list): + i = 0 + for v in doc: + ret.extend(dumpFlat(v, "{}[{}]".format(prefix, i))) + i += 1 + else: + doc = str(doc).replace('\n', '\\n') + ret = [ "{}={}".format(prefix, doc) ] + + return ret From bca67c9dd7a30bfd96085db303a70d46e0a3320f Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 12 Nov 2026 06:39:05 +0100 Subject: [PATCH 02/10] show: move package diff helpers into pkgdiff module Share package inspection and diff rendering without changing their behavior. This prepares the helpers for use while reporting incompatible variants. --- pym/bob/cmds/show.py | 168 +----------------------------------------- pym/bob/pkgdiff.py | 171 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 167 deletions(-) create mode 100644 pym/bob/pkgdiff.py diff --git a/pym/bob/cmds/show.py b/pym/bob/cmds/show.py index e249c19d..788e4d14 100644 --- a/pym/bob/cmds/show.py +++ b/pym/bob/cmds/show.py @@ -5,11 +5,9 @@ from ..errors import ParseError from ..input import RecipeSet -from ..tty import colorize, ADDED, DELETED, DEFAULT, ADDED_HIGHLIGHT, DELETED_HIGHLIGHT -from ..utils import asHexStr +from ..pkgdiff import dumpPackage, diffPackages from .helpers import processDefines, addStandardArgs, dumpYaml, dumpJson, dumpFlat import argparse -import difflib import json import sys import yaml @@ -24,113 +22,6 @@ "shared", ) -def dumpPackage(package): - recipe = package.getRecipe() - - doc = { - "meta" : { - "name" : package.getName(), - "recipe" : recipe.getName(), - "package" : "/".join(package.getStack()), - "deterministic" : package.getPackageStep().isDeterministic(), - }, - "scriptLanguage" : recipe.scriptLanguage.index.value, - "shared" : package.isShared(), - "metaEnvironment" : package.getMetaEnv(), - "relocatable" : package.isRelocatable(), - "jobServer" : recipe.jobServer(), - "root" : recipe.isRoot(), - } - - checkoutStep = package.getCheckoutStep() - if checkoutStep.isValid(): - doc["meta"]["checkoutVariantId"] = asHexStr(checkoutStep.getVariantId()) - doc["checkoutDeterministic"] = checkoutStep.isDeterministic() - doc["checkoutSCM"] = [ - { k:v for k,v in s.getProperties(False, True).items() if not k.startswith("__") } - for s in checkoutStep.getScmList() - ] - doc["checkoutAssert"] = [ - { k:v for k,v in a.items() if not k.startswith("__") } - for a in recipe.checkoutAsserts - ] - doc["checkoutTools"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in checkoutStep.getTools().items() - if name in (checkoutStep.toolDep - checkoutStep.toolDepWeak) - } - doc["checkoutToolsWeak"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in checkoutStep.getTools().items() - if name in checkoutStep.toolDepWeak - } - doc["checkoutVars"] = { - k : v for k, v in checkoutStep.getEnv().items() - if k in recipe.checkoutVars - } - doc["checkoutVarsWeak"] = { - k : v for k, v in checkoutStep.getEnv().items() - if k in recipe.checkoutVarsWeak - } - - buildStep = package.getBuildStep() - if buildStep.isValid(): - doc["meta"]["buildVariantId"] = asHexStr(buildStep.getVariantId()) - doc["buildNetAccess"] = buildStep.hasNetAccess() - doc["depends"] = [ - "/".join(d.getPackage().getStack()) - for d in buildStep.getArguments()[1:] - ] - doc["buildTools"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in buildStep.getTools().items() - if name in (buildStep.toolDep - buildStep.toolDepWeak) - } - doc["buildToolsWeak"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in buildStep.getTools().items() - if name in buildStep.toolDepWeak - } - doc["buildVars"] = { - k : v for k, v in buildStep.getEnv().items() - if k in recipe.buildVars - } - doc["buildVarsWeak"] = { - k : v for k, v in buildStep.getEnv().items() - if k in recipe.buildVarsWeak - } - - packageStep = package.getPackageStep() - doc["meta"]["packageVariantId"] = asHexStr(packageStep.getVariantId()) - doc["packageTools"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in packageStep.getTools().items() - if name in (packageStep.toolDep - packageStep.toolDepWeak) - } - doc["packageToolsWeak"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in packageStep.getTools().items() - if name in packageStep.toolDepWeak - } - doc["packageVars"] = { - k : v for k, v in packageStep.getEnv().items() - if k in recipe.packageVars - } - doc["packageVarsWeak"] = { - k : v for k, v in packageStep.getEnv().items() - if k in recipe.packageVarsWeak - } - doc["packageNetAccess"] = packageStep.hasNetAccess() - doc["fingerprintIf"] = packageStep._isFingerprinted() - - sandbox = packageStep.getSandbox() - if sandbox and sandbox.isEnabled(): - doc["sandbox"] = "/".join(sandbox.getStep().getPackage().getStack()) - else: - doc["sandbox"] = None - - return doc - def filterEmpty(doc): if isinstance(doc, dict): return { k : filterEmpty(v) for k, v in doc.items() if v } @@ -164,63 +55,6 @@ def showPackagesFlat(docs): return "\n".join(ret) -def getSegments(indicator): - mode = False - start = 0 - end = 1 - ret = [] - for c in indicator[2:]: - newMode = c != " " - end += 1 - if mode != newMode: - ret.append((mode, start, end)) - mode = newMode - start = end - ret.append((mode, start, end+1)) - return ret - -def diffPackages(left, right, show_common): - left = [ l+"\n" for l in dumpFlat(left) ] - right = [ l+"\n" for l in dumpFlat(right) ] - diff = [ l for l in difflib.Differ().compare(left, right) - if not l.startswith(" ") or show_common ] - - # Colorize lines with the intraline information from difflib. A bit - # complicated because the affected characters are specified with the next - # line. If the intraline diff is too big then no context is given. - prevLine = origPrevLine = None - ret = [] - for l in diff: - nextLine = origNextLine = l.rstrip() - if nextLine.startswith("? "): - if origPrevLine.startswith("+ "): - codes = { False : ADDED, True : ADDED_HIGHLIGHT } - elif origPrevLine.startswith("- "): - codes = { False : DELETED, True : DELETED_HIGHLIGHT } - else: - codes = { False : DEFAULT, True : DEFAULT } - - res = "" - for mode, start, end in getSegments(nextLine): - res += colorize(origPrevLine[start:end], codes[mode]) - if end < len(origPrevLine): - res += colorize(origPrevLine[end:], codes[False]) - prevLine = res - nextLine = origNextLine = None - elif nextLine.startswith("+ "): - nextLine = colorize(nextLine, ADDED) - elif nextLine.startswith("- "): - nextLine = colorize(nextLine, DELETED) - - if prevLine is not None: ret.append(prevLine) - prevLine = nextLine - origPrevLine = origNextLine - - if prevLine is not None: - ret.append(prevLine) - - return "\n".join(ret) - def doShow(argv, bobRoot): parser = argparse.ArgumentParser(prog="bob show", description="Show properties of one or more packages.") diff --git a/pym/bob/pkgdiff.py b/pym/bob/pkgdiff.py new file mode 100644 index 00000000..26981b8d --- /dev/null +++ b/pym/bob/pkgdiff.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +import difflib +from typing import List, Optional, Tuple +from .tty import colorize, ADDED, DELETED, DEFAULT, ADDED_HIGHLIGHT, DELETED_HIGHLIGHT +from .utils import asHexStr, dumpFlat + +def dumpPackage(package): + recipe = package.getRecipe() + + doc = { + "meta" : { + "name" : package.getName(), + "recipe" : recipe.getName(), + "package" : "/".join(package.getStack()), + "deterministic" : package.getPackageStep().isDeterministic(), + }, + "scriptLanguage" : recipe.scriptLanguage.index.value, + "shared" : package.isShared(), + "metaEnvironment" : package.getMetaEnv(), + "relocatable" : package.isRelocatable(), + "jobServer" : recipe.jobServer(), + "root" : recipe.isRoot(), + } + + checkoutStep = package.getCheckoutStep() + if checkoutStep.isValid(): + doc["meta"]["checkoutVariantId"] = asHexStr(checkoutStep.getVariantId()) + doc["checkoutDeterministic"] = checkoutStep.isDeterministic() + doc["checkoutSCM"] = [ + { k:v for k,v in s.getProperties(False, True).items() if not k.startswith("__") } + for s in checkoutStep.getScmList() + ] + doc["checkoutAssert"] = [ + { k:v for k,v in a.items() if not k.startswith("__") } + for a in recipe.checkoutAsserts + ] + doc["checkoutTools"] = { + name : "/".join(t.getStep().getPackage().getStack()) + for name, t in checkoutStep.getTools().items() + if name in (checkoutStep.toolDep - checkoutStep.toolDepWeak) + } + doc["checkoutToolsWeak"] = { + name : "/".join(t.getStep().getPackage().getStack()) + for name, t in checkoutStep.getTools().items() + if name in checkoutStep.toolDepWeak + } + doc["checkoutVars"] = { + k : v for k, v in checkoutStep.getEnv().items() + if k in recipe.checkoutVars + } + doc["checkoutVarsWeak"] = { + k : v for k, v in checkoutStep.getEnv().items() + if k in recipe.checkoutVarsWeak + } + + buildStep = package.getBuildStep() + if buildStep.isValid(): + doc["meta"]["buildVariantId"] = asHexStr(buildStep.getVariantId()) + doc["buildNetAccess"] = buildStep.hasNetAccess() + doc["depends"] = [ + "/".join(d.getPackage().getStack()) + for d in buildStep.getArguments()[1:] + ] + doc["buildTools"] = { + name : "/".join(t.getStep().getPackage().getStack()) + for name, t in buildStep.getTools().items() + if name in (buildStep.toolDep - buildStep.toolDepWeak) + } + doc["buildToolsWeak"] = { + name : "/".join(t.getStep().getPackage().getStack()) + for name, t in buildStep.getTools().items() + if name in buildStep.toolDepWeak + } + doc["buildVars"] = { + k : v for k, v in buildStep.getEnv().items() + if k in recipe.buildVars + } + doc["buildVarsWeak"] = { + k : v for k, v in buildStep.getEnv().items() + if k in recipe.buildVarsWeak + } + + packageStep = package.getPackageStep() + doc["meta"]["packageVariantId"] = asHexStr(packageStep.getVariantId()) + doc["packageTools"] = { + name : "/".join(t.getStep().getPackage().getStack()) + for name, t in packageStep.getTools().items() + if name in (packageStep.toolDep - packageStep.toolDepWeak) + } + doc["packageToolsWeak"] = { + name : "/".join(t.getStep().getPackage().getStack()) + for name, t in packageStep.getTools().items() + if name in packageStep.toolDepWeak + } + doc["packageVars"] = { + k : v for k, v in packageStep.getEnv().items() + if k in recipe.packageVars + } + doc["packageVarsWeak"] = { + k : v for k, v in packageStep.getEnv().items() + if k in recipe.packageVarsWeak + } + doc["packageNetAccess"] = packageStep.hasNetAccess() + doc["fingerprintIf"] = packageStep._isFingerprinted() + + sandbox = packageStep.getSandbox() + if sandbox and sandbox.isEnabled(): + doc["sandbox"] = "/".join(sandbox.getStep().getPackage().getStack()) + else: + doc["sandbox"] = None + + return doc + +def getSegments(indicator): + mode = False + start = 0 + end = 1 + ret = [] + for c in indicator[2:]: + newMode = c != " " + end += 1 + if mode != newMode: + ret.append((mode, start, end)) + mode = newMode + start = end + ret.append((mode, start, end+1)) + return ret + +def diffPackages(left, right, show_common): + left = [ l+"\n" for l in dumpFlat(left) ] + right = [ l+"\n" for l in dumpFlat(right) ] + diff = [ l for l in difflib.Differ().compare(left, right) + if not l.startswith(" ") or show_common ] + + # Colorize lines with the intraline information from difflib. A bit + # complicated because the affected characters are specified with the next + # line. If the intraline diff is too big then no context is given. + prevLine = origPrevLine = None + ret = [] + for l in diff: + nextLine = origNextLine = l.rstrip() + if nextLine.startswith("? "): + if origPrevLine.startswith("+ "): + codes = { False : ADDED, True : ADDED_HIGHLIGHT } + elif origPrevLine.startswith("- "): + codes = { False : DELETED, True : DELETED_HIGHLIGHT } + else: + codes = { False : DEFAULT, True : DEFAULT } + + res = "" + for mode, start, end in getSegments(nextLine): + res += colorize(origPrevLine[start:end], codes[mode]) + if end < len(origPrevLine): + res += colorize(origPrevLine[end:], codes[False]) + prevLine = res + nextLine = origNextLine = None + elif nextLine.startswith("+ "): + nextLine = colorize(nextLine, ADDED) + elif nextLine.startswith("- "): + nextLine = colorize(nextLine, DELETED) + + if prevLine is not None: ret.append(prevLine) + prevLine = nextLine + origPrevLine = origNextLine + + if prevLine is not None: + ret.append(prevLine) + + return "\n".join(ret) + From b80576ec7ad91190183169d3a30aac7249c32f6d Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 12 Nov 2026 06:43:23 +0100 Subject: [PATCH 03/10] input: explain incompatible package variants Trace differing dependencies to the first concrete package difference and show the comparisons that led there. Resolve provider references with their ambient context so diagnostic paths remain accurate. --- pym/bob/input/recipe.py | 57 +++++++++++++----- pym/bob/pkgdiff.py | 124 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 14 deletions(-) diff --git a/pym/bob/input/recipe.py b/pym/bob/input/recipe.py index eb53ef9d..58edcdaf 100644 --- a/pym/bob/input/recipe.py +++ b/pym/bob/input/recipe.py @@ -11,8 +11,8 @@ PwshLanguage, PythonLanguage from ..stringparser import Env, IfExpression from .common import fetchFingerprintScripts, fetchScripts, mergeScripts, uidGen -from .core import AbstractTool, CoreInterpreter, CoreRef, CoreSandbox -from .package import CorePackage +from .core import AbstractTool, CoreInterpreter, CoreRef, CoreSandbox, PathsConfig +from .package import CorePackage, corePackageInternal from itertools import chain from string import Template import fnmatch @@ -84,7 +84,7 @@ def append(self, ref): self.cache[name] = ref self.ret.append(ref) elif ref2.refGetDestination().variantId != step.variantId: - self.errorHandler(name, self.stack + ref.refGetStack(), self.stack + ref2.refGetStack()) + self.errorHandler(name, self.stack, ref, ref2) def extend(self, gen): for i in gen: self.append(i) @@ -186,6 +186,19 @@ def addTransitiveTools(depSetStrong, depSetWeak, toolsView): depSetStrong |= dependTools depSetWeak |= dependToolsWeak +def _calcRootAmbientDeps(): + """Calculate the ambient dependencies of root packages""" + return ({}, None, {}) + +def _makeDepAmbientCalc(parentAmbientCalc, stack, diffTools, diffSandbox, diffInterpreters): + """Create a resolver for the ambient dependencies (tools, sandbox and + interpreters) of a dependency. """ + def resolve(): + tools, sandbox, interpreters = parentAmbientCalc() + return CoreRef(corePackageInternal, [], diffTools, diffSandbox, + diffInterpreters).refDeref(stack, tools, sandbox, interpreters, PathsConfig(None, False)) + return resolve + class Recipe(object): """Representation of a single recipe @@ -655,7 +668,8 @@ def getPluginProperties(self): return self.__properties def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, - inputTools=Env(), inputInterpreters={}, inputStack=PackageStack(), packageName=None): + inputTools=Env(), inputInterpreters={}, inputStack=PackageStack(), packageName=None, + ambientCalc=_calcRootAmbientDeps): # Cycle detection is based on the recipe package name. Any alias names # are ignored. if self.__packageName in inputStack: @@ -710,7 +724,10 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, subTreePackages = set() directPackages = [] injectedPackages = [] - provideDeps = UniquePackageList(stack.getNameStack(), self.__raiseIncompatibleProvided) + def raiseIncompatible(name, path, left, right): + self.__raiseIncompatibleProvided(name, path, left, right, ambientCalc) + + provideDeps = UniquePackageList(stack.getNameStack(), raiseIncompatible) maybeProvideDeps = [] checkoutDeps = [] results = [] @@ -788,7 +805,9 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, r = self.__recipeSet.getRecipe(recipeName) try: p, s = r.prepare(thisDepEnv, sandboxEnabled, depStates, - thisDepSandbox, thisDepTools, thisDepInterpreters, stack, aliasName) + thisDepSandbox, thisDepTools, thisDepInterpreters, stack, aliasName, + _makeDepAmbientCalc(ambientCalc, stack.getNameStack(), + thisDepDiffTools, thisDepDiffSandbox, thisDepDiffInterpreters)) subTreePackages.add(recipeName) subTreePackages.update(s) depCoreStep = p.getCorePackageStep() @@ -903,9 +922,8 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, if depTrack.prime(): injectedPackages.append(depRef) elif depCoreStep.variantId != depTrack.item.refGetDestination().variantId: - self.__raiseIncompatibleProvided(name, - stack.getNameStack() + depRef.refGetStack(), - stack.getNameStack() + depTrack.item.refGetStack()) + self.__raiseIncompatibleProvided(name, stack.getNameStack(), + depRef, depTrack.item, ambientCalc) if depTrack.useResultOnce(): results.append(depRef) @@ -1134,13 +1152,23 @@ def packageNetAccess(self): else: return self.__packageNetAccess - def __raiseIncompatibleProvided(self, name, stack1, stack2): + def __raiseIncompatibleProvided(self, name, stack, ref1, ref2, ambientCalc): + from ..pkgdiff import calcPackageDiffRecursive, renderPackageDiffChain + pathsConfig = PathsConfig(None, False) + tools, sandbox, interpreters = ambientCalc() + left = ref1.refDeref(stack, tools, sandbox, interpreters, pathsConfig) + right = ref2.refDeref(stack, tools, sandbox, interpreters, pathsConfig) + pair = calcPackageDiffRecursive(left, right) + if pair: + detail = "\nConcrete differences:\n" + renderPackageDiffChain(pair) + else: + detail = "\nNo concrete package difference could be identified." raise ParseError("Incompatible variants of package: {} vs. {}" - .format("/".join(stack1), "/".join(stack2)), + .format("/".join(stack + ref1.refGetStack()), "/".join(stack + ref2.refGetStack())), help= """This error is caused by '{PKG}' that is passed upwards via 'provideDeps' from multiple dependencies of '{CUR}'. These dependencies constitute different variants of '{PKG}' and can therefore not be used in '{CUR}'.""" - .format(PKG=name, CUR=self.__packageName)) + .format(PKG=name, CUR=self.__packageName) + detail) def __raiseIncompatibleLocal(self, r): raise ParseError("Multiple incompatible dependencies to package: {}" @@ -1290,12 +1318,13 @@ def __init__(self, recipeSet, target, packageName, fileName): self.__packageName = packageName self.__source = fileName - def prepare(self, env, sandboxEnabled, states, sandbox, tools, interpreters, stack, packageName=None): + def prepare(self, env, sandboxEnabled, states, sandbox, tools, interpreters, stack, packageName=None, + ambientCalc=_calcRootAmbientDeps): target = env.substitute(self.__target, "alias package") if packageName is None: packageName = self.__packageName return self.__recipeSet.getRecipe(target).prepare(env, sandboxEnabled, states, sandbox, - tools, interpreters, stack, packageName) + tools, interpreters, stack, packageName, ambientCalc) def getPackageName(self): return self.__packageName diff --git a/pym/bob/pkgdiff.py b/pym/bob/pkgdiff.py index 26981b8d..fde49aa4 100644 --- a/pym/bob/pkgdiff.py +++ b/pym/bob/pkgdiff.py @@ -1,10 +1,13 @@ # SPDX-License-Identifier: GPL-3.0-or-later +"""Compare package variant inputs and render their differences.""" + import difflib from typing import List, Optional, Tuple from .tty import colorize, ADDED, DELETED, DEFAULT, ADDED_HIGHLIGHT, DELETED_HIGHLIGHT from .utils import asHexStr, dumpFlat + def dumpPackage(package): recipe = package.getRecipe() @@ -169,3 +172,124 @@ def diffPackages(left, right, show_common): return "\n".join(ret) +def dumpPackagePair(left, right): + """Dump packages, treating equal dependency variants as common values.""" + docs = dumpPackage(left), dumpPackage(right) + for method, field in (("getCheckoutStep", "checkoutTools"), + ("getBuildStep", "buildTools"), + ("getPackageStep", "packageTools")): + a, b = getattr(left, method)(), getattr(right, method)() + ta, tb = a.getTools(), b.getTools() + for name in ta.keys() & tb.keys(): + if ta[name].getStep().getVariantId()[:20] != tb[name].getStep().getVariantId()[:20]: + continue + for key in (field, field + "Weak"): + if name in docs[0].get(key, {}) and name in docs[1].get(key, {}): + docs[1][key][name] = docs[0][key][name] + a, b = left.getBuildStep(), right.getBuildStep() + if a.isValid() and b.isValid(): + for i, (x, y) in enumerate(zip(a.getArguments()[1:], b.getArguments()[1:])): + if x.getVariantId() == y.getVariantId(): + docs[1]["depends"][i] = docs[0]["depends"][i] + a, b = left.getPackageStep().getSandbox(), right.getPackageStep().getSandbox() + if a and b and a.getStep().getVariantId() == b.getStep().getVariantId(): + docs[1]["sandbox"] = docs[0]["sandbox"] + return docs + + +def renderPackageDiffChain(chain, filt=lambda x: x, showCommon=False): + output = [] + for index, (left, right, field) in enumerate(chain): + left, right = left.getPackage(), right.getPackage() + leftDoc, rightDoc = dumpPackagePair(left, right) + leftPath, rightPath = "/".join(left.getStack()), "/".join(right.getStack()) + output.append("diff {} {}\n--- {}\n+++ {}\n{}".format( + leftPath, rightPath, leftPath, rightPath, + diffPackages(filt(leftDoc), filt(rightDoc), showCommon))) + if field: + nextLeft, nextRight, _ = chain[index + 1] + leftName = nextLeft.getPackage().getName() + rightName = nextRight.getPackage().getName() + names = "'{}'".format(leftName) + if leftName != rightName: + names += " and '{}'".format(rightName) + output.append("The above package diff was caused by '{}' from package {}:".format( + field, names)) + return "\n\n".join(output) + + +def calcPackageDiffRecursive(left, right) -> Optional[List[Tuple["Step", "Step", Optional[str]]]]: + """Find a chain of packages that explains a recursive variant difference. + + Each entry contains the left and right steps at that point in the chain and + the field on the preceding package that led to them. The first entry's + field is ``None``. Return ``None`` if no concrete difference can be found. + """ + visited = set() + + def compare(a, b): + aid, bid = a.getVariantId(), b.getVariantId() + key = (aid, bid) + if aid == bid or key in visited: + return None + visited.add(key) + pa, pb = a.getPackage(), b.getPackage() + pending = [] + for sa, sb in zip((pa.getCheckoutStep(), pa.getBuildStep(), pa.getPackageStep()), + (pb.getCheckoutStep(), pb.getBuildStep(), pb.getPackageStep())): + if sa.isValid() != sb.isValid(): + return [(a, b, None)] + if not sa.isValid(): + continue + if sa._coreStep.digestEnv != sb._coreStep.digestEnv: + return [(a, b, None)] + if sa.getLabel() == 'src': + scmA = [s.getProperties(False, True) for s in sa.getScmList()] + scmB = [s.getProperties(False, True) for s in sb.getScmList()] + if scmA != scmB: + return [(a, b, None)] + ta, tb = sa.getTools(), sb.getTools() + if set(ta) != set(tb): + return [(a, b, None)] + for name in sorted(ta): + x, y = ta[name].getStep(), tb[name].getStep() + if x.getVariantId() == y.getVariantId(): + continue + if x._coreStep.corePackage.recipe is not y._coreStep.corePackage.recipe: + return [(a, b, None)] + pending.append((x, y, + {"src": "checkoutTools", + "build": "buildTools", + "dist": "packageTools"}[sa.getLabel()] + "." + name)) + aa = [x for x in sa.getArguments() + if x.isValid() and x.getPackage().getStack() != pa.getStack()] + ab = [x for x in sb.getArguments() + if x.isValid() and x.getPackage().getStack() != pb.getStack()] + if len(aa) != len(ab): + return [(a, b, None)] + for index, (x, y) in enumerate(zip(aa, ab)): + xid, yid = x.getVariantId(), y.getVariantId() + if xid == yid: + continue + if (x.getPackage().getName() != y.getPackage().getName() or + x._coreStep.corePackage.recipe is not y._coreStep.corePackage.recipe): + return [(a, b, None)] + pending.append((x, y, "depends[{}]".format(index))) + x = sa.getSandbox() if sa._isFingerprinted() else None + y = sb.getSandbox() if sb._isFingerprinted() else None + if bool(x) != bool(y): + return [(a, b, None)] + if x and y: + x, y = x.getStep(), y.getStep() + if x.getVariantId() == y.getVariantId(): + continue + if x._coreStep.corePackage.recipe is not y._coreStep.corePackage.recipe: + return [(a, b, None)] + pending.append((x, y, "sandbox")) + for x, y, field in pending: + result = compare(x, y) + if result: + return [(a, b, field)] + result + return None + + return compare(left, right) From a1171a3901ae875fb43f21127acf8e3e46cd3a27 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 12 Nov 2026 06:43:36 +0100 Subject: [PATCH 04/10] test: cover incompatible variant explanations Check that conflicting variants report their differences and retain provider paths through inherited tools. --- test/unit/test_input_recipeset.py | 78 ++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/test/unit/test_input_recipeset.py b/test/unit/test_input_recipeset.py index 7c5747e9..1170cd00 100644 --- a/test/unit/test_input_recipeset.py +++ b/test/unit/test_input_recipeset.py @@ -577,7 +577,83 @@ def testIncompatible(self): recipes = RecipeSet() recipes.parse() packages = recipes.generatePackages(lambda x,y: "unused") - self.assertRaises(ParseError, packages.getRootPackage) + with self.assertRaises(ParseError) as error: + packages.getRootPackage() + message = str(error.exception) + self.assertIn("Concrete differences:", message) + self.assertIn("diff /root/b/c /root/a/c\n--- /root/b/c\n+++ /root/a/c", message) + self.assertIn("buildVars.FOO=A", message) + self.assertIn("buildVars.FOO=B", message) + + def testIncompatibleProvidedToolPaths(self): + self.writeRecipe("root", "root: True\ndepends: [a, b]\npackageScript: 'true'\n") + for name, value in (("a", "first"), ("b", "second")): + self.writeRecipe(name, """\ + depends: + - name: tool + use: [tools] + forward: True + environment: { VARIANT: %s } + - c + provideDeps: [c] + packageScript: 'true' + """ % value) + self.writeRecipe("tool", """\ + packageVars: [VARIANT] + packageScript: 'true' + provideTools: { mytool: bin } + """) + self.writeRecipe("c", "packageTools: [mytool]\npackageScript: 'true'\n") + recipes = RecipeSet() + recipes.parse() + packages = recipes.generatePackages(lambda x, y: "unused") + with self.assertRaises(ParseError) as error: + packages.getRootPackage() + message = str(error.exception) + self.assertIn("caused by 'packageTools.mytool'", message) + self.assertIn("/root/a/tool", message) + self.assertIn("/root/b/tool", message) + self.assertIn("packageVars.VARIANT=first", message) + self.assertIn("packageVars.VARIANT=second", message) + + def testIncompatibleInheritedToolPath(self): + self.writeRecipe("root", """\ + root: True + depends: + - name: tool + use: [tools] + forward: True + environment: { VARIANT: first } + - a + - b + packageScript: 'true' + """) + self.writeRecipe("a", "depends: [c]\nprovideDeps: [c]\npackageScript: 'true'\n") + self.writeRecipe("b", """\ + depends: + - name: tool + use: [tools] + forward: True + environment: { VARIANT: second } + - c + provideDeps: [c] + packageScript: 'true' + """) + self.writeRecipe("tool", """\ + packageVars: [VARIANT] + packageScript: 'true' + provideTools: { mytool: bin } + """) + self.writeRecipe("c", "packageTools: [mytool]\npackageScript: 'true'\n") + recipes = RecipeSet() + recipes.parse() + packages = recipes.generatePackages(lambda x, y: "unused") + with self.assertRaises(ParseError) as error: + packages.getRootPackage() + message = str(error.exception) + self.assertIn("/root/tool", message) + self.assertIn("/root/b/tool", message) + self.assertIn("caused by 'packageTools.mytool'", message) def testProvideDisabled(self): """Providing disabled dependencies is not considered an error.""" From 0d745142a0a2f3d80e0d0a6e34b46ca28cec4f62 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 12 Nov 2026 06:43:47 +0100 Subject: [PATCH 05/10] show: explain variant differences recursively Follow differing dependencies to the first concrete package difference and show the chain of package comparisons. Reuse the regular diff output and its field selection and common-value options. --- contrib/bash-completion/bob | 2 +- pym/bob/cmds/show.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/contrib/bash-completion/bob b/contrib/bash-completion/bob index 0cb8ced0..b1621dc8 100644 --- a/contrib/bash-completion/bob +++ b/contrib/bash-completion/bob @@ -430,7 +430,7 @@ __bob_show() else __bob_complete_path "-D -c --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --show-empty - --show-common --indent --no-indent --format -f" + --show-common --recursive --indent --no-indent --format -f" fi } diff --git a/pym/bob/cmds/show.py b/pym/bob/cmds/show.py index 788e4d14..313f22cf 100644 --- a/pym/bob/cmds/show.py +++ b/pym/bob/cmds/show.py @@ -5,7 +5,7 @@ from ..errors import ParseError from ..input import RecipeSet -from ..pkgdiff import dumpPackage, diffPackages +from ..pkgdiff import diffPackages, dumpPackage, calcPackageDiffRecursive, renderPackageDiffChain from .helpers import processDefines, addStandardArgs, dumpYaml, dumpJson, dumpFlat import argparse import json @@ -77,7 +77,10 @@ def doShow(argv, bobRoot): choices=sorted(ALL_FIELDS), help="Show particular field(s) instead of all") + group.add_argument('--recursive', action='store_true', help='Explain variant differences recursively') args = parser.parse_args(argv) + if args.recursive and args.format != 'diff': + parser.error('--recursive requires diff format') defines = processDefines(args.defines) recipes = RecipeSet() @@ -94,10 +97,12 @@ def doShow(argv, bobRoot): filt = lambda x, filt=filt: filterFields(filt(x), args.field) docs = [] + selected = [] with packages: for p in args.packages: for package in packages.queryPackagePath(p): docs.append(("/".join(package.getStack()), filt(dumpPackage(package)))) + selected.append(package) if not docs: print("Your query matched no packets!", file=sys.stderr) @@ -110,6 +115,12 @@ def doShow(argv, bobRoot): ret = showPackagesFlat(docs) elif len(docs) != 2: raise ParseError("Diff format requires exactly two packages") + elif args.recursive: + pair = calcPackageDiffRecursive(selected[0].getPackageStep(), selected[1].getPackageStep()) + if pair: + ret = renderPackageDiffChain(pair, filt, args.show_common) + else: + ret = "" else: ret = diffPackages(docs[0][1], docs[1][1], args.show_common) From 920f00067d3b18c91bd12c65a22df437ce688290 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 12 Nov 2026 06:43:55 +0100 Subject: [PATCH 06/10] doc: describe recursive show diffs Document how recursive comparisons follow dependencies and apply the selected diff options. --- doc/manpages/bob-show.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/manpages/bob-show.rst b/doc/manpages/bob-show.rst index 1c656f31..201ae19d 100644 --- a/doc/manpages/bob-show.rst +++ b/doc/manpages/bob-show.rst @@ -25,6 +25,7 @@ Synopsis fingerprintIf,jobServer,meta,metaEnvironment,packageNetAccess, packageTools,packageToolsWeak,packageVars,packageVarsWeak, relocatable,root,sandbox,scriptLanguage,shared}] + [--recursive] packages [packages ...] @@ -86,6 +87,13 @@ Options ``--no-sandbox`` Disable sandboxing. +``--recursive`` + With ``--format diff``, find the first package with a direct difference + by following differing dependencies from the same recipe. Display each + package diff along the path and identify the dependency that leads to the + next comparison. The selected fields and output options apply to every + diff. Exactly two packages must be selected. + ``--sandbox`` Enable partial sandboxing. From c0e7885da8edb64749b3e5dd2805fac39334dce5 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 12 Nov 2026 06:44:06 +0100 Subject: [PATCH 07/10] test: cover recursive show diffs Exercise recursive comparisons, dependency equivalence and the supported diff display options. --- test/black-box/show-recursive/config.yaml | 1 + test/black-box/show-recursive/recipes/a.yaml | 1 + test/black-box/show-recursive/recipes/b.yaml | 1 + .../show-recursive/recipes/checkout-left.yaml | 7 ++++ .../recipes/checkout-right.yaml | 7 ++++ .../recipes/common-dep-left.yaml | 7 ++++ .../recipes/common-dep-right.yaml | 7 ++++ .../recipes/fingerprint-left.yaml | 12 +++++++ .../recipes/fingerprint-right.yaml | 12 +++++++ .../show-recursive/recipes/leaf.yaml | 2 ++ .../show-recursive/recipes/middle.yaml | 3 ++ .../show-recursive/recipes/order-left.yaml | 4 +++ .../show-recursive/recipes/order-right.yaml | 4 +++ .../recipes/path-provider-left.yaml | 5 +++ .../recipes/path-provider-right.yaml | 5 +++ .../recipes/recursive-left.yaml | 7 ++++ .../recipes/recursive-right.yaml | 7 ++++ .../recipes/recursive-tool-left.yaml | 8 +++++ .../recipes/recursive-tool-right.yaml | 8 +++++ test/black-box/show-recursive/recipes/s1.yaml | 3 ++ test/black-box/show-recursive/recipes/s2.yaml | 3 ++ .../show-recursive/recipes/sandbox-left.yaml | 7 ++++ .../show-recursive/recipes/sandbox-none.yaml | 4 +++ .../show-recursive/recipes/sandbox-right.yaml | 7 ++++ .../show-recursive/recipes/script-left.yaml | 2 ++ .../show-recursive/recipes/script-right.yaml | 2 ++ .../show-recursive/recipes/tool-absent.yaml | 2 ++ .../recipes/tool-path-left.yaml | 6 ++++ .../recipes/tool-path-right.yaml | 6 ++++ .../show-recursive/recipes/tool.yaml | 5 +++ .../show-recursive/recipes/variant-tool.yaml | 4 +++ test/black-box/show-recursive/run.sh | 33 +++++++++++++++++++ test/black-box/show/run.sh | 6 ++++ 33 files changed, 198 insertions(+) create mode 100644 test/black-box/show-recursive/config.yaml create mode 100644 test/black-box/show-recursive/recipes/a.yaml create mode 100644 test/black-box/show-recursive/recipes/b.yaml create mode 100644 test/black-box/show-recursive/recipes/checkout-left.yaml create mode 100644 test/black-box/show-recursive/recipes/checkout-right.yaml create mode 100644 test/black-box/show-recursive/recipes/common-dep-left.yaml create mode 100644 test/black-box/show-recursive/recipes/common-dep-right.yaml create mode 100644 test/black-box/show-recursive/recipes/fingerprint-left.yaml create mode 100644 test/black-box/show-recursive/recipes/fingerprint-right.yaml create mode 100644 test/black-box/show-recursive/recipes/leaf.yaml create mode 100644 test/black-box/show-recursive/recipes/middle.yaml create mode 100644 test/black-box/show-recursive/recipes/order-left.yaml create mode 100644 test/black-box/show-recursive/recipes/order-right.yaml create mode 100644 test/black-box/show-recursive/recipes/path-provider-left.yaml create mode 100644 test/black-box/show-recursive/recipes/path-provider-right.yaml create mode 100644 test/black-box/show-recursive/recipes/recursive-left.yaml create mode 100644 test/black-box/show-recursive/recipes/recursive-right.yaml create mode 100644 test/black-box/show-recursive/recipes/recursive-tool-left.yaml create mode 100644 test/black-box/show-recursive/recipes/recursive-tool-right.yaml create mode 100644 test/black-box/show-recursive/recipes/s1.yaml create mode 100644 test/black-box/show-recursive/recipes/s2.yaml create mode 100644 test/black-box/show-recursive/recipes/sandbox-left.yaml create mode 100644 test/black-box/show-recursive/recipes/sandbox-none.yaml create mode 100644 test/black-box/show-recursive/recipes/sandbox-right.yaml create mode 100644 test/black-box/show-recursive/recipes/script-left.yaml create mode 100644 test/black-box/show-recursive/recipes/script-right.yaml create mode 100644 test/black-box/show-recursive/recipes/tool-absent.yaml create mode 100644 test/black-box/show-recursive/recipes/tool-path-left.yaml create mode 100644 test/black-box/show-recursive/recipes/tool-path-right.yaml create mode 100644 test/black-box/show-recursive/recipes/tool.yaml create mode 100644 test/black-box/show-recursive/recipes/variant-tool.yaml create mode 100755 test/black-box/show-recursive/run.sh diff --git a/test/black-box/show-recursive/config.yaml b/test/black-box/show-recursive/config.yaml new file mode 100644 index 00000000..39efb6cf --- /dev/null +++ b/test/black-box/show-recursive/config.yaml @@ -0,0 +1 @@ +bobMinimumVersion: '0.25' diff --git a/test/black-box/show-recursive/recipes/a.yaml b/test/black-box/show-recursive/recipes/a.yaml new file mode 100644 index 00000000..7078caf2 --- /dev/null +++ b/test/black-box/show-recursive/recipes/a.yaml @@ -0,0 +1 @@ +packageScript: echo a diff --git a/test/black-box/show-recursive/recipes/b.yaml b/test/black-box/show-recursive/recipes/b.yaml new file mode 100644 index 00000000..b86da3f9 --- /dev/null +++ b/test/black-box/show-recursive/recipes/b.yaml @@ -0,0 +1 @@ +packageScript: echo b diff --git a/test/black-box/show-recursive/recipes/checkout-left.yaml b/test/black-box/show-recursive/recipes/checkout-left.yaml new file mode 100644 index 00000000..b1192456 --- /dev/null +++ b/test/black-box/show-recursive/recipes/checkout-left.yaml @@ -0,0 +1,7 @@ +root: true +checkoutSCM: + scm: git + url: https://example.invalid/left.git + commit: 0123456789abcdef0123456789abcdef01234567 +packageScript: 'true' +buildScript: 'true' diff --git a/test/black-box/show-recursive/recipes/checkout-right.yaml b/test/black-box/show-recursive/recipes/checkout-right.yaml new file mode 100644 index 00000000..ea8ee421 --- /dev/null +++ b/test/black-box/show-recursive/recipes/checkout-right.yaml @@ -0,0 +1,7 @@ +root: true +checkoutSCM: + scm: git + url: https://example.invalid/right.git + commit: abcdef0123456789abcdef0123456789abcdef01 +packageScript: 'true' +buildScript: 'true' diff --git a/test/black-box/show-recursive/recipes/common-dep-left.yaml b/test/black-box/show-recursive/recipes/common-dep-left.yaml new file mode 100644 index 00000000..a90e3db2 --- /dev/null +++ b/test/black-box/show-recursive/recipes/common-dep-left.yaml @@ -0,0 +1,7 @@ +root: true +depends: [a] +buildVars: [VARIANT] +environment: + VARIANT: left +buildScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/common-dep-right.yaml b/test/black-box/show-recursive/recipes/common-dep-right.yaml new file mode 100644 index 00000000..bc67c216 --- /dev/null +++ b/test/black-box/show-recursive/recipes/common-dep-right.yaml @@ -0,0 +1,7 @@ +root: true +depends: [a] +buildVars: [VARIANT] +environment: + VARIANT: right +buildScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/fingerprint-left.yaml b/test/black-box/show-recursive/recipes/fingerprint-left.yaml new file mode 100644 index 00000000..d32819c0 --- /dev/null +++ b/test/black-box/show-recursive/recipes/fingerprint-left.yaml @@ -0,0 +1,12 @@ +root: true +depends: + - name: s1 + use: [sandbox] + forward: true + - name: tool + use: [tools] +packageTools: [tool] +packageVars: [ROOT] +environment: + ROOT: left +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/fingerprint-right.yaml b/test/black-box/show-recursive/recipes/fingerprint-right.yaml new file mode 100644 index 00000000..528831dd --- /dev/null +++ b/test/black-box/show-recursive/recipes/fingerprint-right.yaml @@ -0,0 +1,12 @@ +root: true +depends: + - name: s2 + use: [sandbox] + forward: true + - name: tool + use: [tools] +packageTools: [tool] +packageVars: [ROOT] +environment: + ROOT: right +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/leaf.yaml b/test/black-box/show-recursive/recipes/leaf.yaml new file mode 100644 index 00000000..4f7d6e7a --- /dev/null +++ b/test/black-box/show-recursive/recipes/leaf.yaml @@ -0,0 +1,2 @@ +packageVars: [VARIANT] +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/middle.yaml b/test/black-box/show-recursive/recipes/middle.yaml new file mode 100644 index 00000000..11bf90a3 --- /dev/null +++ b/test/black-box/show-recursive/recipes/middle.yaml @@ -0,0 +1,3 @@ +depends: [leaf] +buildScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/order-left.yaml b/test/black-box/show-recursive/recipes/order-left.yaml new file mode 100644 index 00000000..e863aee5 --- /dev/null +++ b/test/black-box/show-recursive/recipes/order-left.yaml @@ -0,0 +1,4 @@ +root: true +depends: [a, b] +buildScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/order-right.yaml b/test/black-box/show-recursive/recipes/order-right.yaml new file mode 100644 index 00000000..d46050b8 --- /dev/null +++ b/test/black-box/show-recursive/recipes/order-right.yaml @@ -0,0 +1,4 @@ +root: true +depends: [b, a] +buildScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/path-provider-left.yaml b/test/black-box/show-recursive/recipes/path-provider-left.yaml new file mode 100644 index 00000000..56e96c19 --- /dev/null +++ b/test/black-box/show-recursive/recipes/path-provider-left.yaml @@ -0,0 +1,5 @@ +packageScript: 'true' +provideTools: + path-tool: + path: bin/left + libs: [lib/left] diff --git a/test/black-box/show-recursive/recipes/path-provider-right.yaml b/test/black-box/show-recursive/recipes/path-provider-right.yaml new file mode 100644 index 00000000..6cb1d528 --- /dev/null +++ b/test/black-box/show-recursive/recipes/path-provider-right.yaml @@ -0,0 +1,5 @@ +packageScript: 'true' +provideTools: + path-tool: + path: bin/right + libs: [lib/right] diff --git a/test/black-box/show-recursive/recipes/recursive-left.yaml b/test/black-box/show-recursive/recipes/recursive-left.yaml new file mode 100644 index 00000000..2cb0b161 --- /dev/null +++ b/test/black-box/show-recursive/recipes/recursive-left.yaml @@ -0,0 +1,7 @@ +root: true +depends: + - name: middle + environment: + VARIANT: left +buildScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/recursive-right.yaml b/test/black-box/show-recursive/recipes/recursive-right.yaml new file mode 100644 index 00000000..3a756eb6 --- /dev/null +++ b/test/black-box/show-recursive/recipes/recursive-right.yaml @@ -0,0 +1,7 @@ +root: true +depends: + - name: middle + environment: + VARIANT: right +buildScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/recursive-tool-left.yaml b/test/black-box/show-recursive/recipes/recursive-tool-left.yaml new file mode 100644 index 00000000..51da1dbc --- /dev/null +++ b/test/black-box/show-recursive/recipes/recursive-tool-left.yaml @@ -0,0 +1,8 @@ +root: true +depends: + - name: variant-tool + environment: + VARIANT: left + use: [tools] +packageTools: [variant-tool] +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/recursive-tool-right.yaml b/test/black-box/show-recursive/recipes/recursive-tool-right.yaml new file mode 100644 index 00000000..6e521a76 --- /dev/null +++ b/test/black-box/show-recursive/recipes/recursive-tool-right.yaml @@ -0,0 +1,8 @@ +root: true +depends: + - name: variant-tool + environment: + VARIANT: right + use: [tools] +packageTools: [variant-tool] +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/s1.yaml b/test/black-box/show-recursive/recipes/s1.yaml new file mode 100644 index 00000000..d8f599d7 --- /dev/null +++ b/test/black-box/show-recursive/recipes/s1.yaml @@ -0,0 +1,3 @@ +packageScript: echo s1 +provideSandbox: + paths: [/bin] diff --git a/test/black-box/show-recursive/recipes/s2.yaml b/test/black-box/show-recursive/recipes/s2.yaml new file mode 100644 index 00000000..27e62193 --- /dev/null +++ b/test/black-box/show-recursive/recipes/s2.yaml @@ -0,0 +1,3 @@ +packageScript: echo s2 +provideSandbox: + paths: [/bin] diff --git a/test/black-box/show-recursive/recipes/sandbox-left.yaml b/test/black-box/show-recursive/recipes/sandbox-left.yaml new file mode 100644 index 00000000..6137f480 --- /dev/null +++ b/test/black-box/show-recursive/recipes/sandbox-left.yaml @@ -0,0 +1,7 @@ +root: true +depends: + - name: s1 + use: [sandbox] +fingerprintIf: true +fingerprintScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/sandbox-none.yaml b/test/black-box/show-recursive/recipes/sandbox-none.yaml new file mode 100644 index 00000000..9259cc4a --- /dev/null +++ b/test/black-box/show-recursive/recipes/sandbox-none.yaml @@ -0,0 +1,4 @@ +root: true +fingerprintIf: true +fingerprintScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/sandbox-right.yaml b/test/black-box/show-recursive/recipes/sandbox-right.yaml new file mode 100644 index 00000000..79cc4fca --- /dev/null +++ b/test/black-box/show-recursive/recipes/sandbox-right.yaml @@ -0,0 +1,7 @@ +root: true +depends: + - name: s2 + use: [sandbox] +fingerprintIf: true +fingerprintScript: 'true' +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/script-left.yaml b/test/black-box/show-recursive/recipes/script-left.yaml new file mode 100644 index 00000000..c34e50a4 --- /dev/null +++ b/test/black-box/show-recursive/recipes/script-left.yaml @@ -0,0 +1,2 @@ +root: true +packageScript: echo left diff --git a/test/black-box/show-recursive/recipes/script-right.yaml b/test/black-box/show-recursive/recipes/script-right.yaml new file mode 100644 index 00000000..813915dd --- /dev/null +++ b/test/black-box/show-recursive/recipes/script-right.yaml @@ -0,0 +1,2 @@ +root: true +packageScript: echo right diff --git a/test/black-box/show-recursive/recipes/tool-absent.yaml b/test/black-box/show-recursive/recipes/tool-absent.yaml new file mode 100644 index 00000000..e3bacc11 --- /dev/null +++ b/test/black-box/show-recursive/recipes/tool-absent.yaml @@ -0,0 +1,2 @@ +root: true +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/tool-path-left.yaml b/test/black-box/show-recursive/recipes/tool-path-left.yaml new file mode 100644 index 00000000..8db94dfc --- /dev/null +++ b/test/black-box/show-recursive/recipes/tool-path-left.yaml @@ -0,0 +1,6 @@ +root: true +depends: + - name: path-provider-left + use: [tools] +packageTools: [path-tool] +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/tool-path-right.yaml b/test/black-box/show-recursive/recipes/tool-path-right.yaml new file mode 100644 index 00000000..7746cbdf --- /dev/null +++ b/test/black-box/show-recursive/recipes/tool-path-right.yaml @@ -0,0 +1,6 @@ +root: true +depends: + - name: path-provider-right + use: [tools] +packageTools: [path-tool] +packageScript: 'true' diff --git a/test/black-box/show-recursive/recipes/tool.yaml b/test/black-box/show-recursive/recipes/tool.yaml new file mode 100644 index 00000000..627b7ee3 --- /dev/null +++ b/test/black-box/show-recursive/recipes/tool.yaml @@ -0,0 +1,5 @@ +fingerprintIf: true +fingerprintScript: 'true' +packageScript: 'true' +provideTools: + tool: bin diff --git a/test/black-box/show-recursive/recipes/variant-tool.yaml b/test/black-box/show-recursive/recipes/variant-tool.yaml new file mode 100644 index 00000000..c0ff2738 --- /dev/null +++ b/test/black-box/show-recursive/recipes/variant-tool.yaml @@ -0,0 +1,4 @@ +packageVars: [VARIANT] +packageScript: 'true' +provideTools: + variant-tool: bin diff --git a/test/black-box/show-recursive/run.sh b/test/black-box/show-recursive/run.sh new file mode 100755 index 00000000..f8e4be2e --- /dev/null +++ b/test/black-box/show-recursive/run.sh @@ -0,0 +1,33 @@ +#!/bin/bash -e +source "$(dirname "$0")/../../test-lib.sh" "../../.." + +out=$(run_bob show --format diff --recursive recursive-left recursive-right) +printf '%s\n' "$out" | grep -Fx 'diff recursive-left recursive-right' +printf '%s\n' "$out" | grep -Fx -- '--- recursive-left' +printf '%s\n' "$out" | grep -Fx -- '+++ recursive-right' +printf '%s\n' "$out" | grep -F "caused by 'depends[0]'" +printf '%s\n' "$out" | grep -Fx 'diff recursive-left/middle/leaf recursive-right/middle/leaf' +printf '%s\n' "$out" | grep -F -- '- packageVars.VARIANT=left' +printf '%s\n' "$out" | grep -F -- '+ packageVars.VARIANT=right' +out=$(run_bob show --format diff --recursive -f packageVars recursive-left recursive-right) +printf '%s\n' "$out" | grep -F -- '- packageVars.VARIANT=left' +printf '%s\n' "$out" | expect_fail grep -F 'meta.' +out=$(run_bob show --format diff --recursive --show-common -f packageVars recursive-left recursive-right) +printf '%s\n' "$out" | grep -F 'packageVars.VARIANT=right' +out=$(run_bob show --format diff --recursive recursive-tool-left recursive-tool-right) +printf '%s\n' "$out" | grep -F "caused by 'packageTools.variant-tool' from package 'variant-tool':" +printf '%s\n' "$out" | grep -Fx 'diff recursive-tool-left/variant-tool recursive-tool-right/variant-tool' +printf '%s\n' "$out" | grep -F -- '- packageVars.VARIANT=left' +out=$(run_bob show --format diff --recursive order-left order-right) +printf '%s\n' "$out" | grep -F 'depends[0]=order-left/a' +printf '%s\n' "$out" | grep -F 'depends[0]=order-right/b' +out=$(run_bob show --format diff --recursive script-left script-left) +[ -z "$out" ] + +# Equal dependency variants have a common path in the recursive diff. +out=$(run_bob show --format diff --recursive -f depends -f buildVars common-dep-left common-dep-right) +printf '%s\n' "$out" | grep -F -- '- buildVars.VARIANT=left' +printf '%s\n' "$out" | expect_fail grep -F 'depends[0]' +out=$(run_bob show --format diff --recursive --show-common -f depends -f buildVars common-dep-left common-dep-right) +printf '%s\n' "$out" | grep -F ' depends[0]=common-dep-left/a' +printf '%s\n' "$out" | expect_fail grep -F 'depends[0]=common-dep-right/a' diff --git a/test/black-box/show/run.sh b/test/black-box/show/run.sh index 1e338daf..e624cd9f 100755 --- a/test/black-box/show/run.sh +++ b/test/black-box/show/run.sh @@ -44,3 +44,9 @@ assert set(d.keys()) == {"buildVars", "packageVars"} assert set(d["buildVars"].keys()) == {"FOO", "BAR"} assert set(d["packageVars"].keys()) == {"FOO", "BAR", "META"} ' + +# Recursive comparisons accept the normal diff options. +expect_fail --code=2 run_bob show --recursive root/dep root/sandbox +run_bob show --format diff --recursive --show-common root/dep root/sandbox +run_bob show --format diff --recursive -f meta root/dep root/sandbox +expect_fail run_bob show --format diff --recursive root From 87ab9e16e06b1731fd65ac6a2cc44fe4370a879a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 15 Nov 2026 22:53:56 +0100 Subject: [PATCH 08/10] fixup! input: explain incompatible package variants --- pym/bob/pkgdiff.py | 83 +++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/pym/bob/pkgdiff.py b/pym/bob/pkgdiff.py index fde49aa4..8959f1c4 100644 --- a/pym/bob/pkgdiff.py +++ b/pym/bob/pkgdiff.py @@ -4,6 +4,7 @@ import difflib from typing import List, Optional, Tuple +from .input.common import DigestHasher from .tty import colorize, ADDED, DELETED, DEFAULT, ADDED_HIGHLIGHT, DELETED_HIGHLIGHT from .utils import asHexStr, dumpFlat @@ -225,14 +226,20 @@ def calcPackageDiffRecursive(left, right) -> Optional[List[Tuple["Step", "Step", the field on the preceding package that led to them. The first entry's field is ``None``. Return ``None`` if no concrete difference can be found. """ + if left.getVariantId() == right.getVariantId(): + return None + visited = set() - def compare(a, b): - aid, bid = a.getVariantId(), b.getVariantId() - key = (aid, bid) - if aid == bid or key in visited: + # The fingerprint part of the Variant-Id of tools is ignored. + # Consequently, changes to the fingerprint part of tools that are compared + # must be ignored. + def compare(a, b, withFingerprint): + key = a.getVariantId(), b.getVariantId(), withFingerprint + if key in visited: return None visited.add(key) + pa, pb = a.getPackage(), b.getPackage() pending = [] for sa, sb in zip((pa.getCheckoutStep(), pa.getBuildStep(), pa.getPackageStep()), @@ -241,55 +248,61 @@ def compare(a, b): return [(a, b, None)] if not sa.isValid(): continue - if sa._coreStep.digestEnv != sb._coreStep.digestEnv: + + if (sa._coreStep.digestEnv != sb._coreStep.digestEnv or + sa.getDigestScript() != sb.getDigestScript()): return [(a, b, None)] - if sa.getLabel() == 'src': - scmA = [s.getProperties(False, True) for s in sa.getScmList()] - scmB = [s.getProperties(False, True) for s in sb.getScmList()] - if scmA != scmB: - return [(a, b, None)] + ta, tb = sa.getTools(), sb.getTools() + # Strictly speaking, the tool name is ignored for the Variant-Id + # calculation. But different tools will have different Variant-Ids + # in practice. This way it's easier to compare. if set(ta) != set(tb): return [(a, b, None)] for name in sorted(ta): + if (ta[name].getPath() != tb[name].getPath() or + ta[name].getLibs() != tb[name].getLibs()): + return [(a, b, None)] x, y = ta[name].getStep(), tb[name].getStep() - if x.getVariantId() == y.getVariantId(): + # The fingerprint part of tools is ignored! See CoreStep.getDigest(). + xv = DigestHasher.sliceRecipes(x.getVariantId()) + yv = DigestHasher.sliceRecipes(y.getVariantId()) + if xv == yv: continue - if x._coreStep.corePackage.recipe is not y._coreStep.corePackage.recipe: - return [(a, b, None)] - pending.append((x, y, + pending.append((x, y, False, {"src": "checkoutTools", "build": "buildTools", "dist": "packageTools"}[sa.getLabel()] + "." + name)) - aa = [x for x in sa.getArguments() - if x.isValid() and x.getPackage().getStack() != pa.getStack()] - ab = [x for x in sb.getArguments() - if x.isValid() and x.getPackage().getStack() != pb.getStack()] + + # Skip the intra-package arguments in build- and package-steps + firstArg = 0 if sa.isCheckoutStep() else 1 + aa = [x for x in sa.getArguments()[firstArg:] if x.isValid()] + ab = [x for x in sb.getArguments()[firstArg:] if x.isValid()] if len(aa) != len(ab): return [(a, b, None)] for index, (x, y) in enumerate(zip(aa, ab)): xid, yid = x.getVariantId(), y.getVariantId() + if not withFingerprint: + xid, yid = DigestHasher.sliceRecipes(xid), DigestHasher.sliceRecipes(yid) if xid == yid: continue - if (x.getPackage().getName() != y.getPackage().getName() or - x._coreStep.corePackage.recipe is not y._coreStep.corePackage.recipe): - return [(a, b, None)] - pending.append((x, y, "depends[{}]".format(index))) - x = sa.getSandbox() if sa._isFingerprinted() else None - y = sb.getSandbox() if sb._isFingerprinted() else None - if bool(x) != bool(y): - return [(a, b, None)] - if x and y: - x, y = x.getStep(), y.getStep() - if x.getVariantId() == y.getVariantId(): - continue - if x._coreStep.corePackage.recipe is not y._coreStep.corePackage.recipe: + pending.append((x, y, withFingerprint, "depends[{}]".format(index))) + + # The sandbox contributes only to the fingerprint part. + if withFingerprint: + x = sa.getSandbox() if sa._isFingerprinted() else None + y = sb.getSandbox() if sb._isFingerprinted() else None + if bool(x) != bool(y): return [(a, b, None)] - pending.append((x, y, "sandbox")) - for x, y, field in pending: - result = compare(x, y) + if x and y: + x, y = x.getStep(), y.getStep() + if x.getVariantId() != y.getVariantId(): + pending.append((x, y, True, "sandbox")) + + for x, y, withFingerprint, field in pending: + result = compare(x, y, withFingerprint) if result: return [(a, b, field)] + result return None - return compare(left, right) + return compare(left, right, True) From 30c0fec74aaaae18033ef41a33445f91ff0a63ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Sun, 15 Nov 2026 23:08:44 +0100 Subject: [PATCH 09/10] fixup! test: cover incompatible variant explanations --- test/unit/test_input_recipeset.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/unit/test_input_recipeset.py b/test/unit/test_input_recipeset.py index 1170cd00..9c81d777 100644 --- a/test/unit/test_input_recipeset.py +++ b/test/unit/test_input_recipeset.py @@ -655,6 +655,35 @@ def testIncompatibleInheritedToolPath(self): self.assertIn("/root/b/tool", message) self.assertIn("caused by 'packageTools.mytool'", message) + def testIncompatibleProvidedToolPathOnly(self): + """Tools that differ only in their path are reported on the user""" + self.writeRecipe("root", "root: True\ndepends: [a, b]\npackageScript: 'true'\n") + for name, value in (("a", "bin"), ("b", "usr/bin")): + self.writeRecipe(name, """\ + depends: + - name: tool + use: [tools] + forward: True + environment: { TOOLDIR: %s } + - c + provideDeps: [c] + packageScript: 'true' + """ % value) + self.writeRecipe("tool", """\ + packageScript: 'true' + provideTools: { mytool: "${TOOLDIR}" } + """) + self.writeRecipe("c", "packageTools: [mytool]\npackageScript: 'true'\n") + recipes = RecipeSet() + recipes.parse() + packages = recipes.generatePackages(lambda x, y: "unused") + with self.assertRaises(ParseError) as error: + packages.getRootPackage() + message = str(error.exception) + self.assertIn("Concrete differences:", message) + self.assertIn("diff /root/b/c /root/a/c\n--- /root/b/c\n+++ /root/a/c", message) + self.assertNotIn("The above package diff was caused by", message) + def testProvideDisabled(self): """Providing disabled dependencies is not considered an error.""" self.writeRecipe("root", """\ From f3232db5f67c57bb494222a480fac8c3dfe5ecdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kl=C3=B6tzke?= Date: Mon, 16 Nov 2026 18:11:27 +0100 Subject: [PATCH 10/10] show: include tool path and libs The path and library paths of tools are part of the Variant-Id. Show them too, so that variant differences caused by them are visible in the (recursive) package diff. Tools are now shown as map with the providing package, path and libs instead of just the package path. --- doc/manpages/bob-show.rst | 7 +++++ pym/bob/pkgdiff.py | 50 ++++++++++++------------------- test/black-box/show/run.sh | 7 +++++ test/unit/test_input_recipeset.py | 2 ++ 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/doc/manpages/bob-show.rst b/doc/manpages/bob-show.rst index 201ae19d..544611dd 100644 --- a/doc/manpages/bob-show.rst +++ b/doc/manpages/bob-show.rst @@ -53,6 +53,13 @@ the following synthetic properties are shown: * ``sandbox``: The sandbox package that is used for this package. +The used tools (``checkoutTools``, ``buildTools``, ``packageTools`` and their +``*Weak`` counterparts) are shown as map of the tool name to its properties: + +* ``package``: The package that provides the tool +* ``path``: The path of the tool, relative to the package workspace +* ``libs``: The library paths of the tool, relative to the package workspace + By default the information is shown as YAML document. If multiple packages are selected then multiple YAML documents are generated too. The output format can be controlled with the ``--format`` option. The ``flat`` format is a custom diff --git a/pym/bob/pkgdiff.py b/pym/bob/pkgdiff.py index 8959f1c4..47321c17 100644 --- a/pym/bob/pkgdiff.py +++ b/pym/bob/pkgdiff.py @@ -9,6 +9,18 @@ from .utils import asHexStr, dumpFlat +def dumpTools(step, weak): + toolDep = step.toolDepWeak if weak else (step.toolDep - step.toolDepWeak) + return { + name : { + "package" : "/".join(t.getStep().getPackage().getStack()), + "path" : t.getPath(), + "libs" : t.getLibs(), + } + for name, t in step.getTools().items() + if name in toolDep + } + def dumpPackage(package): recipe = package.getRecipe() @@ -39,16 +51,8 @@ def dumpPackage(package): { k:v for k,v in a.items() if not k.startswith("__") } for a in recipe.checkoutAsserts ] - doc["checkoutTools"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in checkoutStep.getTools().items() - if name in (checkoutStep.toolDep - checkoutStep.toolDepWeak) - } - doc["checkoutToolsWeak"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in checkoutStep.getTools().items() - if name in checkoutStep.toolDepWeak - } + doc["checkoutTools"] = dumpTools(checkoutStep, False) + doc["checkoutToolsWeak"] = dumpTools(checkoutStep, True) doc["checkoutVars"] = { k : v for k, v in checkoutStep.getEnv().items() if k in recipe.checkoutVars @@ -66,16 +70,8 @@ def dumpPackage(package): "/".join(d.getPackage().getStack()) for d in buildStep.getArguments()[1:] ] - doc["buildTools"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in buildStep.getTools().items() - if name in (buildStep.toolDep - buildStep.toolDepWeak) - } - doc["buildToolsWeak"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in buildStep.getTools().items() - if name in buildStep.toolDepWeak - } + doc["buildTools"] = dumpTools(buildStep, False) + doc["buildToolsWeak"] = dumpTools(buildStep, True) doc["buildVars"] = { k : v for k, v in buildStep.getEnv().items() if k in recipe.buildVars @@ -87,16 +83,8 @@ def dumpPackage(package): packageStep = package.getPackageStep() doc["meta"]["packageVariantId"] = asHexStr(packageStep.getVariantId()) - doc["packageTools"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in packageStep.getTools().items() - if name in (packageStep.toolDep - packageStep.toolDepWeak) - } - doc["packageToolsWeak"] = { - name : "/".join(t.getStep().getPackage().getStack()) - for name, t in packageStep.getTools().items() - if name in packageStep.toolDepWeak - } + doc["packageTools"] = dumpTools(packageStep, False) + doc["packageToolsWeak"] = dumpTools(packageStep, True) doc["packageVars"] = { k : v for k, v in packageStep.getEnv().items() if k in recipe.packageVars @@ -186,7 +174,7 @@ def dumpPackagePair(left, right): continue for key in (field, field + "Weak"): if name in docs[0].get(key, {}) and name in docs[1].get(key, {}): - docs[1][key][name] = docs[0][key][name] + docs[1][key][name]["package"] = docs[0][key][name]["package"] a, b = left.getBuildStep(), right.getBuildStep() if a.isValid() and b.isValid(): for i, (x, y) in enumerate(zip(a.getArguments()[1:], b.getArguments()[1:])): diff --git a/test/black-box/show/run.sh b/test/black-box/show/run.sh index e624cd9f..8add5e3b 100755 --- a/test/black-box/show/run.sh +++ b/test/black-box/show/run.sh @@ -36,6 +36,13 @@ d = yaml.load(sys.stdin.read(), Loader=yaml.Loader) assert d["checkoutTools"] == {} ' +# Tools are shown with their package, path and libraries +run_bob show root -f buildTools | $PYTHON -c ' +import sys, yaml +d = yaml.load(sys.stdin.read(), Loader=yaml.Loader) +assert d["buildTools"] == {"tool" : {"package" : "root/tool", "path" : "."}} +' + # Verify that filtering works as expected run_bob show root -f buildVars -f packageVars | $PYTHON -c ' import sys, yaml diff --git a/test/unit/test_input_recipeset.py b/test/unit/test_input_recipeset.py index 9c81d777..ca26df20 100644 --- a/test/unit/test_input_recipeset.py +++ b/test/unit/test_input_recipeset.py @@ -683,6 +683,8 @@ def testIncompatibleProvidedToolPathOnly(self): self.assertIn("Concrete differences:", message) self.assertIn("diff /root/b/c /root/a/c\n--- /root/b/c\n+++ /root/a/c", message) self.assertNotIn("The above package diff was caused by", message) + self.assertIn("- packageTools.mytool.path=usr/bin", message) + self.assertIn("+ packageTools.mytool.path=bin", message) def testProvideDisabled(self): """Providing disabled dependencies is not considered an error."""