From 88a99e575ff3b4bf77cc7effc682ccc2a98adb1f Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Tue, 22 Sep 2026 11:47:03 -0300 Subject: [PATCH 1/4] feat(coverity): support retrying archived uploads Coverity can reject initialization after a capture completes. Preserve the source version beside the archive so a later run can upload the same result. Report invalid and failed service responses without leaking jq parse errors. Signed-off-by: Vitor Bandeira --- etc/CodeCoverage.sh | 99 ++++++++++++++++++++++++------- etc/code_coverage_test.py | 120 ++++++++++++++++++++++++++++++++++---- 2 files changed, 188 insertions(+), 31 deletions(-) diff --git a/etc/CodeCoverage.sh b/etc/CodeCoverage.sh index a65d19abc61..f78b7cf5ea7 100755 --- a/etc/CodeCoverage.sh +++ b/etc/CodeCoverage.sh @@ -9,6 +9,7 @@ _help() { usage: $0 [dynamic|test] $0 static $0 static-bazel + $0 upload [VERSION] EOF exit "${1:-1}" @@ -70,7 +71,7 @@ _coverity_bazel() { -- //:openroad } -_coverity() { +_coverity_capture() { "$1" log_file=cov-int/build-log.txt # get compilation coverage percentage @@ -85,41 +86,90 @@ _coverity() { fi tar czvf openroad.tgz cov-int - commitSha="$(git rev-parse HEAD)" + git rev-parse HEAD > openroad.version +} + +_coverity_init_error() { + local response + response=$(tr '\n' ' ' < "$1") + response="${response% }" + if [[ -z ${response} ]]; then + response="empty response" + fi + echo "Coverity build initialization failed: ${response}" >&2 +} + +_coverity_upload() { + local version="${1:-}" + if [[ ! -f openroad.tgz ]]; then + echo "Coverity upload failed: openroad.tgz does not exist." >&2 + return 1 + fi + if [[ -z ${version} && -f openroad.version ]]; then + version=$(< openroad.version) + fi + if [[ -z ${version} ]]; then + echo "Coverity upload failed: no source version is available. Pass VERSION for a legacy archive." >&2 + return 1 + fi if [ -n "${SKIP_COVERITY_UPLOAD+x}" ]; then echo "SKIP_COVERITY_UPLOAD is set. Skipping Coverity upload." - exit 0 + return 0 fi # Step 1: Initialize a build. Fetch a cloud upload url. - curl -X POST \ - -d version="version=${commitSha}" \ - -d description="build=${commitSha}" \ + local response_file + response_file=$(mktemp) + if ! curl --fail-with-body --silent --show-error -X POST \ + -d "version=${version}" \ + -d "description=build=${version}" \ -d email=openroad@ucsd.edu \ - -d token=${token} \ + -d "token=${token}" \ -d file_name=openroad.tgz \ https://scan.coverity.com/projects/21946/builds/init \ - | tee response - - cat response + > "${response_file}"; then + _coverity_init_error "${response_file}" + rm -f "${response_file}" + return 1 + fi # Step 2: Store response data to use in later stages. - # Requires the JSON parsing tool jq. - # If opting for other bash tools, be careful about url encodings. - upload_url=$(jq -r '.url' response) - build_id=$(jq -r '.build_id' response) + local response_fields + if ! response_fields=$(jq -er ' + select( + (.url | type) == "string" + and (.url | length) > 0 + and ((.build_id | type) == "string" or (.build_id | type) == "number") + ) + | [.url, (.build_id | tostring)] + | @tsv + ' "${response_file}" 2>/dev/null); then + _coverity_init_error "${response_file}" + rm -f "${response_file}" + return 1 + fi + local upload_url + local build_id + IFS=$'\t' read -r upload_url build_id <<< "${response_fields}" + rm -f "${response_file}" # Step 3: Upload the tarball to the Cloud. - curl -X PUT \ + if ! curl --fail-with-body --silent --show-error -X PUT \ --header 'Content-Type: application/json' \ --upload-file openroad.tgz \ - "${upload_url}" + "${upload_url}"; then + echo "Coverity archive upload failed." >&2 + return 1 + fi # Step 4: Trigger the build on Scan. - curl -X PUT \ + if ! curl --fail-with-body --silent --show-error -X PUT \ -d "token=${token}" \ - https://scan.coverity.com/projects/21946/builds/${build_id}/enqueue + "https://scan.coverity.com/projects/21946/builds/${build_id}/enqueue"; then + echo "Coverity enqueue failed." >&2 + return 1 + fi } @@ -146,10 +196,19 @@ case "${target}" in fi token="${2}" if [[ ${target} == "static-bazel" ]]; then - _coverity _coverity_bazel + _coverity_capture _coverity_bazel else - _coverity _coverity_cmake + _coverity_capture _coverity_cmake fi + _coverity_upload + ;; + upload ) + if [[ $# -lt 2 || $# -gt 3 ]]; then + echo "'${0} upload' requires a token and accepts one optional version." >&2 + _help + fi + token="${2}" + _coverity_upload "${3:-}" ;; *) echo "invalid argument: ${1}" >&2 diff --git a/etc/code_coverage_test.py b/etc/code_coverage_test.py index 381e9565a40..080e96c4065 100644 --- a/etc/code_coverage_test.py +++ b/etc/code_coverage_test.py @@ -11,10 +11,11 @@ class CodeCoverageTest(unittest.TestCase): def test_static_bazel_uses_uncached_local_capture(self): - result, archive_exists, commands = self._run("static-bazel") + result, archive_exists, version, commands = self._run("static-bazel") self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue(archive_exists) + self.assertEqual(version, "0123456789abcdef\n") self.assertIn("bazelisk clean", commands) self.assertIn("cov-build --dir cov-int --bazel bazelisk build", commands) self.assertIn("--spawn_strategy=local", commands) @@ -25,32 +26,105 @@ def test_static_bazel_uses_uncached_local_capture(self): self.assertNotIn("cmake ", commands) def test_static_keeps_the_cmake_capture(self): - result, archive_exists, commands = self._run("static") + result, archive_exists, version, commands = self._run("static") self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue(archive_exists) + self.assertEqual(version, "0123456789abcdef\n") self.assertIn("cmake -B build .", commands) self.assertIn("cmake --build build", commands) self.assertIn("cov-build --dir cov-int cmake --build build", commands) self.assertNotIn("bazelisk", commands) def test_static_fails_when_capture_percentage_is_missing(self): - result, archive_exists, _ = self._run("static", build_log="no summary\n") + result, archive_exists, version, _ = self._run( + "static", build_log="no summary\n" + ) self.assertEqual(result.returncode, 1) self.assertFalse(archive_exists) + self.assertIsNone(version) self.assertIn("Only got 0%", result.stdout) def test_static_fails_when_capture_percentage_is_low(self): - result, archive_exists, _ = self._run( + result, archive_exists, version, _ = self._run( "static", build_log="Emitted 84 compilation units (84%)\n" ) self.assertEqual(result.returncode, 1) self.assertFalse(archive_exists) + self.assertIsNone(version) self.assertIn("Only got 84%", result.stdout) - def _run(self, mode, build_log="Emitted 100 compilation units (100%)\n"): + def test_upload_reuses_the_archive_without_running_a_capture(self): + result, archive_exists, version, commands = self._run( + "upload", + artifact=True, + version_file="fedcba9876543210\n", + skip_upload=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(archive_exists) + self.assertEqual(version, "fedcba9876543210\n") + self.assertNotIn("cmake", commands) + self.assertNotIn("cov-build", commands) + self.assertIn("version=fedcba9876543210", commands) + self.assertIn("--upload-file openroad.tgz", commands) + self.assertIn("builds/825340/enqueue", commands) + + def test_upload_accepts_a_version_for_a_legacy_archive(self): + result, _, version, commands = self._run( + "upload", + artifact=True, + version_arg="a432b134015160dd", + skip_upload=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIsNone(version) + self.assertIn("version=a432b134015160dd", commands) + + def test_upload_requires_an_archive(self): + result, _, _, commands = self._run( + "upload", version_file="fedcba9876543210\n", skip_upload=False + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("openroad.tgz does not exist", result.stderr) + self.assertNotIn("curl", commands) + + def test_plain_text_initialization_error_is_reported(self): + message = ( + "Your build is already in the queue for analysis. " + "Please wait before uploading another build.\n" + ) + result, archive_exists, version, commands = self._run( + "static", + skip_upload=False, + init_response=message, + ) + + self.assertEqual(result.returncode, 1) + self.assertTrue(archive_exists) + self.assertEqual(version, "0123456789abcdef\n") + self.assertIn( + f"Coverity build initialization failed: {message.strip()}", result.stderr + ) + self.assertNotIn("parse error", result.stderr) + self.assertNotIn("--upload-file", commands) + + def _run( + self, + mode, + build_log="Emitted 100 compilation units (100%)\n", + *, + artifact=False, + version_file=None, + version_arg=None, + skip_upload=True, + init_response='{"url":"https://upload.example/build","build_id":825340}\n', + ): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) etc = root / "repo" / "etc" @@ -84,24 +158,48 @@ def _run(self, mode, build_log="Emitted 100 compilation units (100%)\n"): fake_bin / "git", '#!/bin/sh\nprintf "0123456789abcdef\\n"\n', ) + self._write_executable( + fake_bin / "curl", + """#!/bin/sh +printf 'curl %s\n' "$*" >> "$COMMAND_LOG" +case "$*" in + *builds/init*) printf '%s' "$COVERITY_INIT_RESPONSE" ;; +esac +""", + ) + repo = root / "repo" + if artifact: + (repo / "openroad.tgz").write_bytes(b"coverity archive") + if version_file is not None: + (repo / "openroad.version").write_text(version_file) env = os.environ.copy() env["COMMAND_LOG"] = str(command_log) env["BUILD_LOG_SOURCE"] = str(build_log_source) + env["COVERITY_INIT_RESPONSE"] = init_response env["PATH"] = f"{fake_bin}:{env['PATH']}" - env["SKIP_COVERITY_UPLOAD"] = "1" + if skip_upload: + env["SKIP_COVERITY_UPLOAD"] = "1" + else: + env.pop("SKIP_COVERITY_UPLOAD", None) + + args = [script, mode, "unused-test-token"] + if version_arg is not None: + args.append(version_arg) result = subprocess.run( - [script, mode, "unused-test-token"], - cwd=root / "repo", + args, + cwd=repo, env=env, capture_output=True, text=True, ) - commands = command_log.read_text() - archive = root / "repo" / "openroad.tgz" - return result, archive.is_file(), commands + commands = command_log.read_text() if command_log.exists() else "" + archive = repo / "openroad.tgz" + version_path = repo / "openroad.version" + version = version_path.read_text() if version_path.exists() else None + return result, archive.is_file(), version, commands @staticmethod def _write_executable(path, content): From 6f898cfef4dff157ba4640fc2bec84c0c1a282a6 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Fri, 2 Oct 2026 15:45:30 -0300 Subject: [PATCH 2/4] fix(coverity): support curl versions before 7.76 curl added --fail-with-body in 7.76. Ubuntu 20.04 has curl 7.68, so the upload failed before it sent a request. The initialization request now has no --fail flag. An HTTP error body is not valid init JSON, so the jq check in Step 2 rejects it and reports the body. The upload and enqueue requests use --fail. Signed-off-by: Vitor Bandeira --- etc/CodeCoverage.sh | 8 +++++--- etc/code_coverage_test.py | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/etc/CodeCoverage.sh b/etc/CodeCoverage.sh index f78b7cf5ea7..86809689880 100755 --- a/etc/CodeCoverage.sh +++ b/etc/CodeCoverage.sh @@ -119,9 +119,11 @@ _coverity_upload() { fi # Step 1: Initialize a build. Fetch a cloud upload url. + # Omit --fail so an HTTP error body reaches the jq check in Step 2 and is + # reported. --fail-with-body needs curl 7.76, which Ubuntu 20.04 lacks. local response_file response_file=$(mktemp) - if ! curl --fail-with-body --silent --show-error -X POST \ + if ! curl --silent --show-error -X POST \ -d "version=${version}" \ -d "description=build=${version}" \ -d email=openroad@ucsd.edu \ @@ -155,7 +157,7 @@ _coverity_upload() { rm -f "${response_file}" # Step 3: Upload the tarball to the Cloud. - if ! curl --fail-with-body --silent --show-error -X PUT \ + if ! curl --fail --silent --show-error -X PUT \ --header 'Content-Type: application/json' \ --upload-file openroad.tgz \ "${upload_url}"; then @@ -164,7 +166,7 @@ _coverity_upload() { fi # Step 4: Trigger the build on Scan. - if ! curl --fail-with-body --silent --show-error -X PUT \ + if ! curl --fail --silent --show-error -X PUT \ -d "token=${token}" \ "https://scan.coverity.com/projects/21946/builds/${build_id}/enqueue"; then echo "Coverity enqueue failed." >&2 diff --git a/etc/code_coverage_test.py b/etc/code_coverage_test.py index 080e96c4065..3daf5e3244a 100644 --- a/etc/code_coverage_test.py +++ b/etc/code_coverage_test.py @@ -72,6 +72,8 @@ def test_upload_reuses_the_archive_without_running_a_capture(self): self.assertIn("version=fedcba9876543210", commands) self.assertIn("--upload-file openroad.tgz", commands) self.assertIn("builds/825340/enqueue", commands) + # curl 7.68 (Ubuntu 20.04) does not support --fail-with-body. + self.assertNotIn("--fail-with-body", commands) def test_upload_accepts_a_version_for_a_legacy_archive(self): result, _, version, commands = self._run( From 6af9319cfc9f466d1827ac8a0bfb53161b26ecf6 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Fri, 2 Oct 2026 15:46:07 -0300 Subject: [PATCH 3/4] fix(coverity): require jq before an upload The jq call hides its stderr. When jq was missing, the upload reported "empty response" instead of the real cause. The check runs after the skip test, so a scan-only run does not need jq. Signed-off-by: Vitor Bandeira --- etc/CodeCoverage.sh | 5 +++++ etc/code_coverage_test.py | 25 ++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/etc/CodeCoverage.sh b/etc/CodeCoverage.sh index 86809689880..cac6aed54b8 100755 --- a/etc/CodeCoverage.sh +++ b/etc/CodeCoverage.sh @@ -117,6 +117,11 @@ _coverity_upload() { echo "SKIP_COVERITY_UPLOAD is set. Skipping Coverity upload." return 0 fi + # Check after the skip so a scan-only run does not need jq. + if ! command -v jq >/dev/null 2>&1; then + echo "Coverity upload failed: jq is required." >&2 + return 1 + fi # Step 1: Initialize a build. Fetch a cloud upload url. # Omit --fail so an HTTP error body reaches the jq check in Step 2 and is diff --git a/etc/code_coverage_test.py b/etc/code_coverage_test.py index 3daf5e3244a..ad8b8ee5805 100644 --- a/etc/code_coverage_test.py +++ b/etc/code_coverage_test.py @@ -96,6 +96,19 @@ def test_upload_requires_an_archive(self): self.assertIn("openroad.tgz does not exist", result.stderr) self.assertNotIn("curl", commands) + def test_upload_requires_jq(self): + result, _, _, commands = self._run( + "upload", + artifact=True, + version_file="fedcba9876543210\n", + skip_upload=False, + hide_jq=True, + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("jq is required", result.stderr) + self.assertNotIn("curl", commands) + def test_plain_text_initialization_error_is_reported(self): message = ( "Your build is already in the queue for analysis. " @@ -125,6 +138,7 @@ def _run( version_file=None, version_arg=None, skip_upload=True, + hide_jq=False, init_response='{"url":"https://upload.example/build","build_id":825340}\n', ): with tempfile.TemporaryDirectory() as tmp: @@ -179,7 +193,16 @@ def _run( env["COMMAND_LOG"] = str(command_log) env["BUILD_LOG_SOURCE"] = str(build_log_source) env["COVERITY_INIT_RESPONSE"] = init_response - env["PATH"] = f"{fake_bin}:{env['PATH']}" + if hide_jq: + # Expose only the tools that the upload path runs before + # it needs jq. + tools = root / "tools" + tools.mkdir() + for name in ("bash", "env", "dirname", "readlink"): + (tools / name).symlink_to(shutil.which(name)) + env["PATH"] = f"{fake_bin}:{tools}" + else: + env["PATH"] = f"{fake_bin}:{env['PATH']}" if skip_upload: env["SKIP_COVERITY_UPLOAD"] = "1" else: From 69459c7b01f83b0b5d0fb8347b710727cf61b9d9 Mon Sep 17 00:00:00 2001 From: Vitor Bandeira Date: Fri, 2 Oct 2026 15:49:27 -0300 Subject: [PATCH 4/4] test(coverity): skip jq-dependent tests without jq The Bazel CI image does not install jq. Two upload tests failed there because the script could not parse a valid reply. A third test passed only by accident. Skip these three tests when jq is missing. The test for the missing-jq error still runs on all hosts. Signed-off-by: Vitor Bandeira --- etc/code_coverage_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/etc/code_coverage_test.py b/etc/code_coverage_test.py index ad8b8ee5805..ea9c4df629c 100644 --- a/etc/code_coverage_test.py +++ b/etc/code_coverage_test.py @@ -8,6 +8,10 @@ import unittest from pathlib import Path +# The upload parses the Coverity reply with jq. The Bazel CI image does not +# install jq, so tests that reach that parse run only where jq exists. +requires_jq = unittest.skipUnless(shutil.which("jq"), "jq is not installed") + class CodeCoverageTest(unittest.TestCase): def test_static_bazel_uses_uncached_local_capture(self): @@ -56,6 +60,7 @@ def test_static_fails_when_capture_percentage_is_low(self): self.assertIsNone(version) self.assertIn("Only got 84%", result.stdout) + @requires_jq def test_upload_reuses_the_archive_without_running_a_capture(self): result, archive_exists, version, commands = self._run( "upload", @@ -75,6 +80,7 @@ def test_upload_reuses_the_archive_without_running_a_capture(self): # curl 7.68 (Ubuntu 20.04) does not support --fail-with-body. self.assertNotIn("--fail-with-body", commands) + @requires_jq def test_upload_accepts_a_version_for_a_legacy_archive(self): result, _, version, commands = self._run( "upload", @@ -109,6 +115,7 @@ def test_upload_requires_jq(self): self.assertIn("jq is required", result.stderr) self.assertNotIn("curl", commands) + @requires_jq def test_plain_text_initialization_error_is_reported(self): message = ( "Your build is already in the queue for analysis. "