diff --git a/contrib/bash-completion/bob b/contrib/bash-completion/bob index 11222065b..0cb8ced04 100644 --- a/contrib/bash-completion/bob +++ b/contrib/bash-completion/bob @@ -128,7 +128,7 @@ __bob_cook() elif [[ "$prev" = "--always-checkout" ]] ; then COMPREPLY=( ) else - __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --clean-checkout --attic --no-attic" + __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --clean-checkout --check-deps --no-check-deps --attic --no-attic" fi } diff --git a/doc/manpages/bob-build-dev.rst b/doc/manpages/bob-build-dev.rst index 9574a068a..bdf07ce96 100644 --- a/doc/manpages/bob-build-dev.rst +++ b/doc/manpages/bob-build-dev.rst @@ -101,6 +101,29 @@ Options This is the default unless the user changed it in ``default.yaml``. +``--check-deps`` + Verify that steps do not modify the workspaces of their dependencies. + + Steps must only ever change their own workspace. All other workspaces that + are visible to a step (e.g. the sources of the package, its dependencies + and tools) must be left untouched. Inside a sandbox this is enforced by + mounting these workspaces read-only. When a step is executed without a + sandbox, Bob cannot prevent such modifications. With this option, Bob will + hash all dependency workspaces before and after executing a step outside + of a sandbox and fail the build if any of them was modified. + + Modified build and package workspaces are invalidated and will be rebuilt + on the next invocation. Steps that are executed in a sandbox are not + checked because they cannot modify their dependencies anyway. + + This is the default for release mode builds (*bob build*). See + ``--no-check-deps`` for the inverse option. + + .. note:: + If more than one step is executed concurrently (``-j``), the error might + be attributed to the wrong step because they may share common + dependencies. + ``--clean`` Do clean builds by clearing the build directory before executing the build commands. It will *not* clean all build results (e.g. like ``make clean``) @@ -192,6 +215,11 @@ Options ``--link-deps`` Create symlinks to dependencies next to workspace. +``--no-check-deps`` + Do not verify that steps leave the workspaces of their dependencies + untouched. This is the default for development mode builds (*bob dev*). + See ``--check-deps`` for details. + ``--no-install`` Do not install shared packages if a shared location is configured. diff --git a/doc/manpages/bob-build.rst b/doc/manpages/bob-build.rst index aa88335a6..724899384 100644 --- a/doc/manpages/bob-build.rst +++ b/doc/manpages/bob-build.rst @@ -26,6 +26,7 @@ Synopsis [--install | --no-install] [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--clean-checkout] [--attic | --no-attic] + [--check-deps | --no-check-deps] PACKAGE [PACKAGE ...] Description @@ -51,5 +52,6 @@ overridden individually by their inverse switches: * ``--download=yes`` * ``--clean`` * ``--sandbox`` +* ``--check-deps`` .. include:: bob-build-dev.rst diff --git a/doc/manpages/bob-dev.rst b/doc/manpages/bob-dev.rst index 23fadc4c5..be218e14c 100644 --- a/doc/manpages/bob-dev.rst +++ b/doc/manpages/bob-dev.rst @@ -26,6 +26,7 @@ Synopsis [--install | --no-install] [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--clean-checkout] [--attic | --no-attic] + [--check-deps | --no-check-deps] PACKAGE [PACKAGE ...] Description @@ -51,6 +52,7 @@ overridden individually by their inverse switches: * ``--download=deps`` * ``--incremental`` * ``--no-sandbox`` +* ``--no-check-deps`` Source code checkout -------------------- diff --git a/doc/manpages/bob-jenkins.rst b/doc/manpages/bob-jenkins.rst index 652e3e990..d31890433 100644 --- a/doc/manpages/bob-jenkins.rst +++ b/doc/manpages/bob-jenkins.rst @@ -475,6 +475,17 @@ audit.meta. ``meta.`` to select artifacts built by this project. Variables that are defined by Bob itself (e.g. ``meta.jenkins-node``) cannot be redefined! +jobs.check-deps + Boolean option (possible values: '0' or 'false' resp. '1' or 'true') that + verifies that steps do not modify the workspaces of their dependencies. + This option is enabled by default. All dependency workspaces of a step that + is executed outside of a sandbox are hashed before and after the step. The + job fails if any of them was modified. Steps that are executed in a sandbox + are not checked because they cannot modify their dependencies anyway. + + You may disable this option to slightly improve build times. Usually the + impact is not noticeable, though. If in doubt, leave it enabled. + jobs.clean.post-build Controls whether the workspace of the Jenkins jobs should be cleaned after a job was run. By default, the workspace is kept (``never``). Use the other diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index 59c4a36d3..3b3ebc920 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -3039,6 +3039,7 @@ attic ``--[no]-attic`` Boolean audit ``--[no]-audit`` Boolean build_mode ``-b`` | ``-B`` | String (``normal``, ``build-only`` or ``--normal`` ``checkout-only``) +check_deps ``--[no-]check-deps`` Boolean clean ``--clean`` | Boolean ``--incremental`` clean_checkout ``--clean-checkout`` Boolean diff --git a/pym/bob/builder.py b/pym/bob/builder.py index 8d90a13fd..38d7567e2 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -58,6 +58,66 @@ async def hashWorkspace(step): # Cache is also used by share module return await loop.run_in_executor(None, hashDirectory, step.getStoragePath(), cachePath) +class DependencyChecker: + """Detect modifications of dependency workspaces by a step. + + All workspaces that are visible to a step, except its own one, must not be + modified. This is enforced by a sandbox where these workspaces are mounted + read-only. If a step is executed without a sandbox, this class can be used + to detect inadvertent modifications by hashing the workspaces before and + after the step was executed. + """ + + def __init__(self, step): + # The checkout and build step of the same package are always visible + # to the package step. Stop at checkout steps because they might have + # dependencies due to the 'checkoutDep' flag. Mirrors the logic of + # the sandbox mounts in StepSpec.fromStep(). + deps = [ d for d in step.getAllDepSteps() if d.isValid() ] + extra = step + while extra.isValid() and not extra.isCheckoutStep() and len(extra.getArguments()) > 0: + extra = extra.getArguments()[0] + if extra.isValid(): + deps.append(extra) + + self.__deps = {} + for d in deps: + self.__deps.setdefault(d.getWorkspacePath(), d) + self.__hashes = None + + async def __hashAll(self): + hashes = await asyncio.gather(*(hashWorkspace(d) for d in self.__deps.values())) + return dict(zip(self.__deps.keys(), hashes)) + + async def prepare(self): + self.__hashes = await self.__hashAll() + + async def verify(self): + """Verify that no dependency workspace was modified. + + Changed workspaces of build and package steps are invalidated so that + they are executed again on the next invocation. The checkout steps + are always re-hashed and need no special treatment. + """ + after = await self.__hashAll() + changed = sorted(path for path, h in after.items() + if self.__hashes[path] != h) + if not changed: + return + + for path in changed: + if self.__deps[path].isCheckoutStep(): continue + BobState().delInputHashes(path) + BobState().setResultHash(path, datetime.datetime.now()) + + raise BuildError("Step modified the following dependency workspace(s): " + + ", ".join(changed), + help="Steps must not change the workspaces of their dependencies. " + "Fix the recipe to work only in its own workspace. Changed build " + "and package workspaces will be rebuilt on the next " + "invocation. Use 'bob status' to inspect changed source " + "workspaces. You may disable this check with '--no-check-deps'.") + class HashOnce: def __init__(self, step): self.step = step @@ -493,6 +553,7 @@ def __init__(self, verbose, force, skipDeps, buildOnly, preserveEnv, self.__executor = None self.__attic = True self.__slimSandbox = False + self.__checkDeps = False def setExecutor(self, executor): self.__executor = executor @@ -591,6 +652,14 @@ def setAtticEnable(self, enable): def setSlimSandbox(self, enable): self.__slimSandbox = enable + def setCheckDependencies(self, enable): + """Verify that dependency workspaces are not modified by a step. + + Only applies to steps that are not executed in a sandbox. Sandboxed + steps cannot modify their dependencies anyway. + """ + self.__checkDeps = enable + def setShareHandler(self, handler): self.__share = handler @@ -879,6 +948,11 @@ async def _runShell(self, step, scriptName, logger, workspaceCreated, cleanWorks executor=self.__executor) if step.jobServer() and self.__jobServer: invoker.setMakeJobserver(self.__jobServer.getSubMakeJobserverCfg(step.jobServer())) + if self.__checkDeps and not spec.hasSandbox: + depChecker = DependencyChecker(step) + await depChecker.prepare() + else: + depChecker = None ret = await invoker.executeStep(mode, workspaceCreated, cleanWorkspace) if not self.__bufferedStdIO: ttyReinit() # work around MSYS2 messing up the console if ret == -int(signal.SIGINT): @@ -891,6 +965,8 @@ async def _runShell(self, step, scriptName, logger, workspaceCreated, cleanWorks .format(absRunFile, ret), help="You may resume at this point with '--resume' after fixing the error.", returncode=ret) + if depChecker is not None: + await depChecker.verify() def getStatistic(self): return self.__statistic diff --git a/pym/bob/cmds/build/build.py b/pym/bob/cmds/build/build.py index 85b764d79..9ad67f080 100644 --- a/pym/bob/cmds/build/build.py +++ b/pym/bob/cmds/build/build.py @@ -232,6 +232,11 @@ def _downloadLayerArgument(arg): parser.add_argument('--clean-checkout', action='store_true', default=None, dest='clean_checkout', help="Do a clean checkout if SCM state is dirty.") group = parser.add_mutually_exclusive_group() + group.add_argument('--check-deps', action='store_true', default=None, dest='check_deps', + help="Fail if a step outside a sandbox modifies the workspaces of its dependencies.") + group.add_argument('--no-check-deps', action='store_false', dest='check_deps', + help="Do not check if steps modify the workspaces of their dependencies.") + group = parser.add_mutually_exclusive_group() group.add_argument('--attic', action='store_true', default=None, help="Move scm to attic if inline switch is not possible (default).") group.add_argument('--no-attic', action='store_false', default=None, dest='attic', @@ -270,6 +275,7 @@ def _downloadLayerArgument(arg): 'download' : "deps" if develop else "yes", 'sandbox' : "no" if develop else "yes", 'clean_checkout' : False, + 'check_deps' : not develop, 'no_logfiles' : False, 'link_deps' : True, 'keep_going' : False, @@ -328,6 +334,7 @@ def _downloadLayerArgument(arg): builder.setLocalDownloadMode(args.download) builder.setLocalDownloadLayerMode(args.download_layer) builder.setCleanCheckout(args.clean_checkout) + builder.setCheckDependencies(args.check_deps) builder.setAlwaysCheckout(args.always_checkout + cfg.get('always_checkout', [])) builder.setLinkDependencies(args.link_deps) builder.setJobserverConfig(jobserverCfg) diff --git a/pym/bob/cmds/jenkins/exec.py b/pym/bob/cmds/jenkins/exec.py index b07f42f4d..b9da9602b 100644 --- a/pym/bob/cmds/jenkins/exec.py +++ b/pym/bob/cmds/jenkins/exec.py @@ -31,6 +31,7 @@ def __init__(self, specFile): self.__recipesAudit = [] self.__execIR = [] self.slimSandbox = False + self.checkDeps = True with open(specFile, "r") as f: f.readline() # skip shebang @@ -68,6 +69,8 @@ def __handleCfg(self, line): self.scmAlwaysCheckout = isTrue(v) elif k == "slimSandbox": self.slimSandbox = isTrue(v) + elif k == "check-deps": + self.checkDeps = isTrue(v) else: raise AssertionError(line) @@ -241,6 +244,7 @@ def doJenkinsExecuteRun(argv, bobRoot): if spec.scmAlwaysCheckout: builder.setAlwaysCheckout([".*"]) builder.setSlimSandbox(spec.slimSandbox) + builder.setCheckDependencies(spec.checkDeps) builder.cook(ir.getRoots(), False, loop) return 0 diff --git a/pym/bob/cmds/jenkins/jenkins.py b/pym/bob/cmds/jenkins/jenkins.py index c5041c34d..5c5447a66 100644 --- a/pym/bob/cmds/jenkins/jenkins.py +++ b/pym/bob/cmds/jenkins/jenkins.py @@ -448,6 +448,10 @@ def dumpXML(self, orig, config, date, recipesAudit=None): ]) if config.sharedQuota: execCmds.append("quota=" + config.sharedQuota) + # Only record the non-default case. Otherwise all existing jobs would + # be reconfigured and scheduled again. + if not config.jobsCheckDeps: + execCmds.append("check-deps=0") if recipesAudit: execCmds.extend(["[recipes-audit]", recipesAudit]) auditMeta = config.getAuditMeta() diff --git a/pym/bob/input/recipeset.py b/pym/bob/input/recipeset.py index 8c2c2e5bc..dc585203d 100644 --- a/pym/bob/input/recipeset.py +++ b/pym/bob/input/recipeset.py @@ -112,6 +112,7 @@ class RecipeSet: schema.Optional('attic') : bool, schema.Optional('audit') : bool, schema.Optional('build_mode') : schema.Or("build-only","normal", "checkout-only"), + schema.Optional('check_deps') : bool, schema.Optional('clean') : bool, schema.Optional('clean_checkout') : bool, schema.Optional('destination') : str, diff --git a/pym/bob/state.py b/pym/bob/state.py index d9e36c5f3..65d374e71 100644 --- a/pym/bob/state.py +++ b/pym/bob/state.py @@ -154,6 +154,9 @@ def setOption(self, key, value, errorHandler): import re if not re.fullmatch(r"[0-9A-Za-z._-]+", key): errorHandler("Invalid audit meta variable name: " + key) + elif key == "jobs.check-deps": + if value.lower() not in ("0", "false", "1", "true"): + errorHandler("jobs.check-deps must be any of: 0/false/1/true") elif key == "jobs.clean.post-build": if value not in ("never", "on-success", "on-failure", "always"): errorHandler("jobs.clean.post-build must be any of: never/on-success/on-failure/always") @@ -239,6 +242,10 @@ def urlPassword(self, password): def artifactsCopy(self): return self.__options.get("artifacts.copy", "jenkins") + @property + def jobsCheckDeps(self): + return isTrue(self.__options.get("jobs.check-deps", "1")) + @property def jobsIsolate(self): return self.__options.get("jobs.isolate") diff --git a/test/black-box/check-deps/recipes/dep.yaml b/test/black-box/check-deps/recipes/dep.yaml new file mode 100644 index 000000000..bb45635ce --- /dev/null +++ b/test/black-box/check-deps/recipes/dep.yaml @@ -0,0 +1,2 @@ +packageScript: | + echo dep > result.txt diff --git a/test/black-box/check-deps/recipes/root.yaml b/test/black-box/check-deps/recipes/root.yaml new file mode 100644 index 000000000..bab150318 --- /dev/null +++ b/test/black-box/check-deps/recipes/root.yaml @@ -0,0 +1,21 @@ +root: True +depends: + - dep + +checkoutScript: | + echo src > src.txt + +buildVars: [MODIFY] +buildScript: | + case "${MODIFY:-}" in + src) echo bad >> "$1/src.txt" ;; + dep) echo bad >> "$2/result.txt" ;; + esac + cp "$2/result.txt" dep.txt + +packageVars: [MODIFY_PKG] +packageScript: | + if [[ ${MODIFY_PKG:-} == build ]] ; then + echo bad >> "$1/dep.txt" + fi + cp "$1/dep.txt" . diff --git a/test/black-box/check-deps/run.sh b/test/black-box/check-deps/run.sh new file mode 100755 index 000000000..9276634da --- /dev/null +++ b/test/black-box/check-deps/run.sh @@ -0,0 +1,35 @@ +#!/bin/bash -e +# +# Verify that modifications of dependency workspaces are detected +# (--check-deps) when steps are executed outside of a sandbox. +# +source "$(dirname "$0")/../../test-lib.sh" "../../.." +cleanup + +# Well behaving steps pass in release mode where the check is the default +run_bob build root +expect_equal "dep" "$(cat "$(run_bob query-path -f '{dist}' --release root)/dep.txt")" + +# Modifying a dependency or the own sources fails by default in release mode +expect_fail run_bob build root -DMODIFY=dep +expect_fail run_bob build root -DMODIFY=src +expect_fail run_bob build root -DMODIFY_PKG=build + +# The modified dependency must have been invalidated and is rebuilt +run_bob build root +expect_equal "dep" "$(cat "$(run_bob query-path -f '{dist}' --release root/dep)/result.txt")" +expect_equal "dep" "$(cat "$(run_bob query-path -f '{dist}' --release root)/dep.txt")" + +# The check can be disabled +run_bob build --no-check-deps root -DMODIFY=dep + +# The check is disabled by default in development mode but can be enabled +run_bob dev root -DMODIFY=dep +expect_fail run_bob dev --check-deps root -DMODIFY=src +expect_fail run_bob dev --check-deps root -DMODIFY=dep +run_bob dev --check-deps root +expect_equal "dep" "$(cat "$(run_bob query-path -f '{dist}' --develop root/dep)/result.txt")" +expect_equal "dep" "$(cat "$(run_bob query-path -f '{dist}' --develop root)/dep.txt")" + +# Make sure the check also works in parallel builds +expect_fail run_bob dev --check-deps -j4 root -DMODIFY=dep diff --git a/test/unit/test_jenkins_build.py b/test/unit/test_jenkins_build.py index aceb982bc..abb26e251 100644 --- a/test/unit/test_jenkins_build.py +++ b/test/unit/test_jenkins_build.py @@ -194,6 +194,45 @@ def testIncremental(self): self.assertEqual(f.read(), "2\n") +class JenkinsCheckDeps(JenkinsTests, TestCase): + """Verify that modifications of dependency workspaces are detected""" + + def setUp(self): + super().setUp() + try: + self.writeRecipe("lib", """\ + packageScript: | + echo lib > result.txt + """) + self.writeRecipe("root", """\ + root: True + depends: [ "lib" ] + buildScript: | + echo bad >> "$2/result.txt" + cp "$2/result.txt" . + packageScript: | + cp "$1/result.txt" . + """) + except: + super().tearDown() + raise + + def testDefaultEnabled(self): + """Modifying a dependency fails the build by default""" + self.executeBobJenkinsCmd("push test") + with self.assertRaises(subprocess.CalledProcessError): + self.jenkinsMock.run() + + def testDisabled(self): + """The check can be disabled by an extended option""" + self.executeBobJenkinsCmd("set-options test -o jobs.check-deps=0") + self.executeBobJenkinsCmd("push test") + self.jenkinsMock.run() + with self.getJobResult("root") as d: + with open(os.path.join(d, "result.txt")) as f: + self.assertEqual(f.read(), "lib\nbad\n") + + class JenkinsProvideInterpreters(JenkinsTests, TestCase): """Verify that provideInterpreters is honored by Jenkins builds""" diff --git a/test/unit/test_jenkins_set_options.py b/test/unit/test_jenkins_set_options.py index a4f488840..a4fc1cf18 100644 --- a/test/unit/test_jenkins_set_options.py +++ b/test/unit/test_jenkins_set_options.py @@ -99,6 +99,16 @@ def testSandboxModes(self): self.executeBobJenkinsCmd("set-options test --strict-sandbox") self.assertEqual(BobState().getJenkinsConfig("test").sandbox.mode, "strict") + def testCheckDeps(self): + self.assertTrue(BobState().getJenkinsConfig("test").jobsCheckDeps) + self.executeBobJenkinsCmd("set-options test -o jobs.check-deps=0") + self.assertFalse(BobState().getJenkinsConfig("test").jobsCheckDeps) + self.executeBobJenkinsCmd("set-options test -o jobs.check-deps=true") + self.assertTrue(BobState().getJenkinsConfig("test").jobsCheckDeps) + + with self.assertRaises(SystemExit): + self.executeBobJenkinsCmd("set-options test -o jobs.check-deps=foo") + def testPostBuildClean(self): self.executeBobJenkinsCmd("set-options test -o jobs.clean.post-build=never") self.assertFalse(BobState().getJenkinsConfig("test").postBuildCleanSuccess)