diff --git a/etc/CodeCoverage.sh b/etc/CodeCoverage.sh index a65d19abc61..cac6aed54b8 100755 --- a/etc/CodeCoverage.sh +++ b/etc/CodeCoverage.sh @@ -9,6 +9,7 @@ _help() { usage: $0 [dynamic|test] $0 static $0 static-bazel + $0 upload [VERSION] EOF exit "${1:-1}" @@ -70,7 +71,7 @@ _coverity_bazel() { -- //:openroad } -_coverity() { +_coverity_capture() { "$1" log_file=cov-int/build-log.txt # get compilation coverage percentage @@ -85,41 +86,97 @@ _coverity() { fi tar czvf openroad.tgz cov-int - commitSha="$(git rev-parse HEAD)" + git rev-parse HEAD > openroad.version +} + +_coverity_init_error() { + local response + response=$(tr '\n' ' ' < "$1") + response="${response% }" + if [[ -z ${response} ]]; then + response="empty response" + fi + echo "Coverity build initialization failed: ${response}" >&2 +} + +_coverity_upload() { + local version="${1:-}" + if [[ ! -f openroad.tgz ]]; then + echo "Coverity upload failed: openroad.tgz does not exist." >&2 + return 1 + fi + if [[ -z ${version} && -f openroad.version ]]; then + version=$(< openroad.version) + fi + if [[ -z ${version} ]]; then + echo "Coverity upload failed: no source version is available. Pass VERSION for a legacy archive." >&2 + return 1 + fi if [ -n "${SKIP_COVERITY_UPLOAD+x}" ]; then echo "SKIP_COVERITY_UPLOAD is set. Skipping Coverity upload." - exit 0 + return 0 + fi + # Check after the skip so a scan-only run does not need jq. + if ! command -v jq >/dev/null 2>&1; then + echo "Coverity upload failed: jq is required." >&2 + return 1 fi # Step 1: Initialize a build. Fetch a cloud upload url. - curl -X POST \ - -d version="version=${commitSha}" \ - -d description="build=${commitSha}" \ + # Omit --fail so an HTTP error body reaches the jq check in Step 2 and is + # reported. --fail-with-body needs curl 7.76, which Ubuntu 20.04 lacks. + local response_file + response_file=$(mktemp) + if ! curl --silent --show-error -X POST \ + -d "version=${version}" \ + -d "description=build=${version}" \ -d email=openroad@ucsd.edu \ - -d token=${token} \ + -d "token=${token}" \ -d file_name=openroad.tgz \ https://scan.coverity.com/projects/21946/builds/init \ - | tee response - - cat response + > "${response_file}"; then + _coverity_init_error "${response_file}" + rm -f "${response_file}" + return 1 + fi # Step 2: Store response data to use in later stages. - # Requires the JSON parsing tool jq. - # If opting for other bash tools, be careful about url encodings. - upload_url=$(jq -r '.url' response) - build_id=$(jq -r '.build_id' response) + local response_fields + if ! response_fields=$(jq -er ' + select( + (.url | type) == "string" + and (.url | length) > 0 + and ((.build_id | type) == "string" or (.build_id | type) == "number") + ) + | [.url, (.build_id | tostring)] + | @tsv + ' "${response_file}" 2>/dev/null); then + _coverity_init_error "${response_file}" + rm -f "${response_file}" + return 1 + fi + local upload_url + local build_id + IFS=$'\t' read -r upload_url build_id <<< "${response_fields}" + rm -f "${response_file}" # Step 3: Upload the tarball to the Cloud. - curl -X PUT \ + if ! curl --fail --silent --show-error -X PUT \ --header 'Content-Type: application/json' \ --upload-file openroad.tgz \ - "${upload_url}" + "${upload_url}"; then + echo "Coverity archive upload failed." >&2 + return 1 + fi # Step 4: Trigger the build on Scan. - curl -X PUT \ + if ! curl --fail --silent --show-error -X PUT \ -d "token=${token}" \ - https://scan.coverity.com/projects/21946/builds/${build_id}/enqueue + "https://scan.coverity.com/projects/21946/builds/${build_id}/enqueue"; then + echo "Coverity enqueue failed." >&2 + return 1 + fi } @@ -146,10 +203,19 @@ case "${target}" in fi token="${2}" if [[ ${target} == "static-bazel" ]]; then - _coverity _coverity_bazel + _coverity_capture _coverity_bazel else - _coverity _coverity_cmake + _coverity_capture _coverity_cmake fi + _coverity_upload + ;; + upload ) + if [[ $# -lt 2 || $# -gt 3 ]]; then + echo "'${0} upload' requires a token and accepts one optional version." >&2 + _help + fi + token="${2}" + _coverity_upload "${3:-}" ;; *) echo "invalid argument: ${1}" >&2 diff --git a/etc/code_coverage_test.py b/etc/code_coverage_test.py index 381e9565a40..ea9c4df629c 100644 --- a/etc/code_coverage_test.py +++ b/etc/code_coverage_test.py @@ -8,13 +8,18 @@ import unittest from pathlib import Path +# The upload parses the Coverity reply with jq. The Bazel CI image does not +# install jq, so tests that reach that parse run only where jq exists. +requires_jq = unittest.skipUnless(shutil.which("jq"), "jq is not installed") + class CodeCoverageTest(unittest.TestCase): def test_static_bazel_uses_uncached_local_capture(self): - result, archive_exists, commands = self._run("static-bazel") + result, archive_exists, version, commands = self._run("static-bazel") self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue(archive_exists) + self.assertEqual(version, "0123456789abcdef\n") self.assertIn("bazelisk clean", commands) self.assertIn("cov-build --dir cov-int --bazel bazelisk build", commands) self.assertIn("--spawn_strategy=local", commands) @@ -25,32 +30,124 @@ def test_static_bazel_uses_uncached_local_capture(self): self.assertNotIn("cmake ", commands) def test_static_keeps_the_cmake_capture(self): - result, archive_exists, commands = self._run("static") + result, archive_exists, version, commands = self._run("static") self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue(archive_exists) + self.assertEqual(version, "0123456789abcdef\n") self.assertIn("cmake -B build .", commands) self.assertIn("cmake --build build", commands) self.assertIn("cov-build --dir cov-int cmake --build build", commands) self.assertNotIn("bazelisk", commands) def test_static_fails_when_capture_percentage_is_missing(self): - result, archive_exists, _ = self._run("static", build_log="no summary\n") + result, archive_exists, version, _ = self._run( + "static", build_log="no summary\n" + ) self.assertEqual(result.returncode, 1) self.assertFalse(archive_exists) + self.assertIsNone(version) self.assertIn("Only got 0%", result.stdout) def test_static_fails_when_capture_percentage_is_low(self): - result, archive_exists, _ = self._run( + result, archive_exists, version, _ = self._run( "static", build_log="Emitted 84 compilation units (84%)\n" ) self.assertEqual(result.returncode, 1) self.assertFalse(archive_exists) + self.assertIsNone(version) self.assertIn("Only got 84%", result.stdout) - def _run(self, mode, build_log="Emitted 100 compilation units (100%)\n"): + @requires_jq + def test_upload_reuses_the_archive_without_running_a_capture(self): + result, archive_exists, version, commands = self._run( + "upload", + artifact=True, + version_file="fedcba9876543210\n", + skip_upload=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(archive_exists) + self.assertEqual(version, "fedcba9876543210\n") + self.assertNotIn("cmake", commands) + self.assertNotIn("cov-build", commands) + self.assertIn("version=fedcba9876543210", commands) + self.assertIn("--upload-file openroad.tgz", commands) + self.assertIn("builds/825340/enqueue", commands) + # curl 7.68 (Ubuntu 20.04) does not support --fail-with-body. + self.assertNotIn("--fail-with-body", commands) + + @requires_jq + def test_upload_accepts_a_version_for_a_legacy_archive(self): + result, _, version, commands = self._run( + "upload", + artifact=True, + version_arg="a432b134015160dd", + skip_upload=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIsNone(version) + self.assertIn("version=a432b134015160dd", commands) + + def test_upload_requires_an_archive(self): + result, _, _, commands = self._run( + "upload", version_file="fedcba9876543210\n", skip_upload=False + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("openroad.tgz does not exist", result.stderr) + self.assertNotIn("curl", commands) + + def test_upload_requires_jq(self): + result, _, _, commands = self._run( + "upload", + artifact=True, + version_file="fedcba9876543210\n", + skip_upload=False, + hide_jq=True, + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("jq is required", result.stderr) + self.assertNotIn("curl", commands) + + @requires_jq + def test_plain_text_initialization_error_is_reported(self): + message = ( + "Your build is already in the queue for analysis. " + "Please wait before uploading another build.\n" + ) + result, archive_exists, version, commands = self._run( + "static", + skip_upload=False, + init_response=message, + ) + + self.assertEqual(result.returncode, 1) + self.assertTrue(archive_exists) + self.assertEqual(version, "0123456789abcdef\n") + self.assertIn( + f"Coverity build initialization failed: {message.strip()}", result.stderr + ) + self.assertNotIn("parse error", result.stderr) + self.assertNotIn("--upload-file", commands) + + def _run( + self, + mode, + build_log="Emitted 100 compilation units (100%)\n", + *, + artifact=False, + version_file=None, + version_arg=None, + skip_upload=True, + hide_jq=False, + init_response='{"url":"https://upload.example/build","build_id":825340}\n', + ): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) etc = root / "repo" / "etc" @@ -84,24 +181,57 @@ def _run(self, mode, build_log="Emitted 100 compilation units (100%)\n"): fake_bin / "git", '#!/bin/sh\nprintf "0123456789abcdef\\n"\n', ) + self._write_executable( + fake_bin / "curl", + """#!/bin/sh +printf 'curl %s\n' "$*" >> "$COMMAND_LOG" +case "$*" in + *builds/init*) printf '%s' "$COVERITY_INIT_RESPONSE" ;; +esac +""", + ) + repo = root / "repo" + if artifact: + (repo / "openroad.tgz").write_bytes(b"coverity archive") + if version_file is not None: + (repo / "openroad.version").write_text(version_file) env = os.environ.copy() env["COMMAND_LOG"] = str(command_log) env["BUILD_LOG_SOURCE"] = str(build_log_source) - env["PATH"] = f"{fake_bin}:{env['PATH']}" - env["SKIP_COVERITY_UPLOAD"] = "1" + env["COVERITY_INIT_RESPONSE"] = init_response + if hide_jq: + # Expose only the tools that the upload path runs before + # it needs jq. + tools = root / "tools" + tools.mkdir() + for name in ("bash", "env", "dirname", "readlink"): + (tools / name).symlink_to(shutil.which(name)) + env["PATH"] = f"{fake_bin}:{tools}" + else: + env["PATH"] = f"{fake_bin}:{env['PATH']}" + if skip_upload: + env["SKIP_COVERITY_UPLOAD"] = "1" + else: + env.pop("SKIP_COVERITY_UPLOAD", None) + + args = [script, mode, "unused-test-token"] + if version_arg is not None: + args.append(version_arg) result = subprocess.run( - [script, mode, "unused-test-token"], - cwd=root / "repo", + args, + cwd=repo, env=env, capture_output=True, text=True, ) - commands = command_log.read_text() - archive = root / "repo" / "openroad.tgz" - return result, archive.is_file(), commands + commands = command_log.read_text() if command_log.exists() else "" + archive = repo / "openroad.tgz" + version_path = repo / "openroad.version" + version = version_path.read_text() if version_path.exists() else None + return result, archive.is_file(), version, commands @staticmethod def _write_executable(path, content):