feat(coverity): support retrying archived uploads

etc/CodeCoverage.sh

@@ -9,6 +9,7 @@ _help() {
 usage: $0 [dynamic|test]
        $0 static <TOKEN>
        $0 static-bazel <TOKEN>
+       $0 upload <TOKEN> [VERSION]
 
 EOF
     exit "${1:-1}"
@@ -70,7 +71,7 @@ _coverity_bazel() {
         -- //:openroad
 }
 
-_coverity() {
+_coverity_capture() {
     "$1"
     log_file=cov-int/build-log.txt
     # get compilation coverage percentage
@@ -85,41 +86,97 @@ _coverity() {
     fi
 
     tar czvf openroad.tgz cov-int
-    commitSha="$(git rev-parse HEAD)"
+    git rev-parse HEAD > openroad.version
+}
+
+_coverity_init_error() {
+    local response
+    response=$(tr '\n' ' ' < "$1")
+    response="${response% }"
+    if [[ -z ${response} ]]; then
+        response="empty response"
+    fi
+    echo "Coverity build initialization failed: ${response}" >&2
+}
+
+_coverity_upload() {
+    local version="${1:-}"
+    if [[ ! -f openroad.tgz ]]; then
+        echo "Coverity upload failed: openroad.tgz does not exist." >&2
+        return 1
+    fi
+    if [[ -z ${version} && -f openroad.version ]]; then
+        version=$(< openroad.version)
+    fi
+    if [[ -z ${version} ]]; then
+        echo "Coverity upload failed: no source version is available. Pass VERSION for a legacy archive." >&2
+        return 1
+    fi
 
     if [ -n "${SKIP_COVERITY_UPLOAD+x}" ]; then
         echo "SKIP_COVERITY_UPLOAD is set. Skipping Coverity upload."
-        exit 0
+        return 0
+    fi
+    # Check after the skip so a scan-only run does not need jq.
+    if ! command -v jq >/dev/null 2>&1; then
+        echo "Coverity upload failed: jq is required." >&2
+        return 1
     fi
 
     # Step 1: Initialize a build. Fetch a cloud upload url.
-    curl -X POST \
-        -d version="version=${commitSha}" \
-        -d description="build=${commitSha}" \
+    # Omit --fail so an HTTP error body reaches the jq check in Step 2 and is
+    # reported. --fail-with-body needs curl 7.76, which Ubuntu 20.04 lacks.
+    local response_file
+    response_file=$(mktemp)
+    if ! curl --silent --show-error -X POST \
+        -d "version=${version}" \
+        -d "description=build=${version}" \
         -d email=openroad@ucsd.edu \
-        -d token=${token} \
+        -d "token=${token}" \
         -d file_name=openroad.tgz \
         https://scan.coverity.com/projects/21946/builds/init \
-        | tee response
-
-    cat response
+        > "${response_file}"; then
+        _coverity_init_error "${response_file}"
+        rm -f "${response_file}"
+        return 1
+    fi
 
     # Step 2: Store response data to use in later stages.
-    # Requires the JSON parsing tool jq.
-    # If opting for other bash tools, be careful about url encodings.
-    upload_url=$(jq -r '.url' response)
-    build_id=$(jq -r '.build_id' response)
+    local response_fields
+    if ! response_fields=$(jq -er '
+        select(
+            (.url | type) == "string"
+            and (.url | length) > 0
+            and ((.build_id | type) == "string" or (.build_id | type) == "number")
+        )
+        | [.url, (.build_id | tostring)]
+        | @tsv
+    ' "${response_file}" 2>/dev/null); then
+        _coverity_init_error "${response_file}"
+        rm -f "${response_file}"
+        return 1
+    fi
+    local upload_url
+    local build_id
+    IFS=$'\t' read -r upload_url build_id <<< "${response_fields}"
+    rm -f "${response_file}"
 
     # Step 3: Upload the tarball to the Cloud.
-    curl -X PUT \
+    if ! curl --fail --silent --show-error -X PUT \
         --header 'Content-Type: application/json' \
         --upload-file openroad.tgz \
-        "${upload_url}"
+        "${upload_url}"; then
+        echo "Coverity archive upload failed." >&2
+        return 1
+    fi
 
     # Step 4: Trigger the build on Scan.
-    curl -X PUT \
+    if ! curl --fail --silent --show-error -X PUT \
         -d "token=${token}" \
-        https://scan.coverity.com/projects/21946/builds/${build_id}/enqueue
+        "https://scan.coverity.com/projects/21946/builds/${build_id}/enqueue"; then
+        echo "Coverity enqueue failed." >&2
+        return 1
+    fi
 
 }
 
@@ -146,10 +203,19 @@ case "${target}" in
         fi
         token="${2}"
         if [[ ${target} == "static-bazel" ]]; then
-            _coverity _coverity_bazel
+            _coverity_capture _coverity_bazel
         else
-            _coverity _coverity_cmake
+            _coverity_capture _coverity_cmake
         fi
+        _coverity_upload
+        ;;
+    upload )
+        if [[ $# -lt 2 || $# -gt 3 ]]; then
+            echo "'${0} upload' requires a token and accepts one optional version." >&2
+            _help
+        fi
+        token="${2}"
+        _coverity_upload "${3:-}"
         ;;
     *)
         echo "invalid argument: ${1}" >&2

etc/code_coverage_test.py

@@ -8,13 +8,18 @@
 import unittest
 from pathlib import Path
 
+# The upload parses the Coverity reply with jq. The Bazel CI image does not
+# install jq, so tests that reach that parse run only where jq exists.
+requires_jq = unittest.skipUnless(shutil.which("jq"), "jq is not installed")
+
 
 class CodeCoverageTest(unittest.TestCase):
     def test_static_bazel_uses_uncached_local_capture(self):
-        result, archive_exists, commands = self._run("static-bazel")
+        result, archive_exists, version, commands = self._run("static-bazel")
 
         self.assertEqual(result.returncode, 0, result.stderr)
         self.assertTrue(archive_exists)
+        self.assertEqual(version, "0123456789abcdef\n")
         self.assertIn("bazelisk clean", commands)
         self.assertIn("cov-build --dir cov-int --bazel bazelisk build", commands)
         self.assertIn("--spawn_strategy=local", commands)
@@ -25,32 +30,124 @@ def test_static_bazel_uses_uncached_local_capture(self):
         self.assertNotIn("cmake ", commands)
 
     def test_static_keeps_the_cmake_capture(self):
-        result, archive_exists, commands = self._run("static")
+        result, archive_exists, version, commands = self._run("static")
 
         self.assertEqual(result.returncode, 0, result.stderr)
         self.assertTrue(archive_exists)
+        self.assertEqual(version, "0123456789abcdef\n")
         self.assertIn("cmake -B build .", commands)
         self.assertIn("cmake --build build", commands)
         self.assertIn("cov-build --dir cov-int cmake --build build", commands)
         self.assertNotIn("bazelisk", commands)
 
     def test_static_fails_when_capture_percentage_is_missing(self):
-        result, archive_exists, _ = self._run("static", build_log="no summary\n")
+        result, archive_exists, version, _ = self._run(
+            "static", build_log="no summary\n"
+        )
 
         self.assertEqual(result.returncode, 1)
         self.assertFalse(archive_exists)
+        self.assertIsNone(version)
         self.assertIn("Only got 0%", result.stdout)
 
     def test_static_fails_when_capture_percentage_is_low(self):
-        result, archive_exists, _ = self._run(
+        result, archive_exists, version, _ = self._run(
             "static", build_log="Emitted 84 compilation units (84%)\n"
         )
 
         self.assertEqual(result.returncode, 1)
         self.assertFalse(archive_exists)
+        self.assertIsNone(version)
         self.assertIn("Only got 84%", result.stdout)
 
-    def _run(self, mode, build_log="Emitted 100 compilation units (100%)\n"):
+    @requires_jq
+    def test_upload_reuses_the_archive_without_running_a_capture(self):
+        result, archive_exists, version, commands = self._run(
+            "upload",
+            artifact=True,
+            version_file="fedcba9876543210\n",
+            skip_upload=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertTrue(archive_exists)
+        self.assertEqual(version, "fedcba9876543210\n")
+        self.assertNotIn("cmake", commands)
+        self.assertNotIn("cov-build", commands)
+        self.assertIn("version=fedcba9876543210", commands)
+        self.assertIn("--upload-file openroad.tgz", commands)
+        self.assertIn("builds/825340/enqueue", commands)
+        # curl 7.68 (Ubuntu 20.04) does not support --fail-with-body.
+        self.assertNotIn("--fail-with-body", commands)
+
+    @requires_jq
+    def test_upload_accepts_a_version_for_a_legacy_archive(self):
+        result, _, version, commands = self._run(
+            "upload",
+            artifact=True,
+            version_arg="a432b134015160dd",
+            skip_upload=False,
+        )
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIsNone(version)
+        self.assertIn("version=a432b134015160dd", commands)
+
+    def test_upload_requires_an_archive(self):
+        result, _, _, commands = self._run(
+            "upload", version_file="fedcba9876543210\n", skip_upload=False
+        )
+
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("openroad.tgz does not exist", result.stderr)
+        self.assertNotIn("curl", commands)
+
+    def test_upload_requires_jq(self):
+        result, _, _, commands = self._run(
+            "upload",
+            artifact=True,
+            version_file="fedcba9876543210\n",
+            skip_upload=False,
+            hide_jq=True,
+        )
+
+        self.assertEqual(result.returncode, 1)
+        self.assertIn("jq is required", result.stderr)
+        self.assertNotIn("curl", commands)
+
+    @requires_jq
+    def test_plain_text_initialization_error_is_reported(self):
+        message = (
+            "Your build is already in the queue for analysis. "
+            "Please wait before uploading another build.\n"
+        )
+        result, archive_exists, version, commands = self._run(
+            "static",
+            skip_upload=False,
+            init_response=message,
+        )
+
+        self.assertEqual(result.returncode, 1)
+        self.assertTrue(archive_exists)
+        self.assertEqual(version, "0123456789abcdef\n")
+        self.assertIn(
+            f"Coverity build initialization failed: {message.strip()}", result.stderr
+        )
+        self.assertNotIn("parse error", result.stderr)
+        self.assertNotIn("--upload-file", commands)
+
+    def _run(
+        self,
+        mode,
+        build_log="Emitted 100 compilation units (100%)\n",
+        *,
+        artifact=False,
+        version_file=None,
+        version_arg=None,
+        skip_upload=True,
+        hide_jq=False,
+        init_response='{"url":"https://upload.example/build","build_id":825340}\n',
+    ):
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
             etc = root / "repo" / "etc"
@@ -84,24 +181,57 @@ def _run(self, mode, build_log="Emitted 100 compilation units (100%)\n"):
                 fake_bin / "git",
                 '#!/bin/sh\nprintf "0123456789abcdef\\n"\n',
             )
+            self._write_executable(
+                fake_bin / "curl",
+                """#!/bin/sh
+printf 'curl %s\n' "$*" >> "$COMMAND_LOG"
+case "$*" in
+    *builds/init*) printf '%s' "$COVERITY_INIT_RESPONSE" ;;
+esac
+""",
+            )
+            repo = root / "repo"
+            if artifact:
+                (repo / "openroad.tgz").write_bytes(b"coverity archive")
+            if version_file is not None:
+                (repo / "openroad.version").write_text(version_file)
 
             env = os.environ.copy()
             env["COMMAND_LOG"] = str(command_log)
             env["BUILD_LOG_SOURCE"] = str(build_log_source)
-            env["PATH"] = f"{fake_bin}:{env['PATH']}"
-            env["SKIP_COVERITY_UPLOAD"] = "1"
+            env["COVERITY_INIT_RESPONSE"] = init_response
+            if hide_jq:
+                # Expose only the tools that the upload path runs before
+                # it needs jq.
+                tools = root / "tools"
+                tools.mkdir()
+                for name in ("bash", "env", "dirname", "readlink"):
+                    (tools / name).symlink_to(shutil.which(name))
+                env["PATH"] = f"{fake_bin}:{tools}"
+            else:
+                env["PATH"] = f"{fake_bin}:{env['PATH']}"
+            if skip_upload:
+                env["SKIP_COVERITY_UPLOAD"] = "1"
+            else:
+                env.pop("SKIP_COVERITY_UPLOAD", None)
+
+            args = [script, mode, "unused-test-token"]
+            if version_arg is not None:
+                args.append(version_arg)
 
             result = subprocess.run(
-                [script, mode, "unused-test-token"],
-                cwd=root / "repo",
+                args,
+                cwd=repo,
                 env=env,
                 capture_output=True,
                 text=True,
             )
 
-            commands = command_log.read_text()
-            archive = root / "repo" / "openroad.tgz"
-            return result, archive.is_file(), commands
+            commands = command_log.read_text() if command_log.exists() else ""
+            archive = repo / "openroad.tgz"
+            version_path = repo / "openroad.version"
+            version = version_path.read_text() if version_path.exists() else None
+            return result, archive.is_file(), version, commands
 
     @staticmethod
     def _write_executable(path, content):