ci: make transient infra failures retry quickly

.gitlab/generate-appsec.php

@@ -160,11 +160,12 @@
     - switch-php $SWITCH_PHP_VERSION
     - cd appsec/build
     - if [[ "$SWITCH_PHP_VERSION" == *"asan"* ]]; then ASAN_FLAG=ON; else ASAN_FLAG=OFF; fi
-    - "cmake .. -DCMAKE_BUILD_TYPE=Debug
+    # FetchContent downloads boost and rapidjson at configure time: a Depot 503 or a boost transfer error there exits 75 and is retried.
+    - "../../.gitlab/run-with-retryable-download.sh cmake .. -DCMAKE_BUILD_TYPE=Debug
       -DCMAKE_CXX_FLAGS='-stdlib=libc++' -DCMAKE_CXX_LINK_FLAGS='-stdlib=libc++'
 	  -DDD_APPSEC_TESTING=ON -DBOOST_CACHE_PREFIX=$CI_PROJECT_DIR/boost-cache
-      -DENABLE_ASAN=$ASAN_FLAG"
-    - ASAN_OPTIONS=malloc_context_size=0 make -j 4 xtest
+      -DENABLE_ASAN=$ASAN_FLAG || exit $?"
+    - ASAN_OPTIONS=malloc_context_size=0 ../../.gitlab/run-with-retryable-download.sh make -j 4 xtest || exit $?
 
 .appsec_integration_tests:
   stage: test
@@ -193,7 +194,8 @@
         TERM=dumb ./gradlew loadCaches --info
       fi
 
-      TERM=dumb ./gradlew $targets --info -Pbuildscan --scan -PcheckCoreDumps
+      # Gradle forwards the appsec CMake output: a Depot 503 in its FetchContent exits 75, which is retried.
+      TERM=dumb ../../../.gitlab/run-with-retryable-download.sh ./gradlew $targets --info -Pbuildscan --scan -PcheckCoreDumps || exit $?
       TERM=dumb ./gradlew saveCaches --info
   after_script:
     - mkdir -p "${CI_PROJECT_DIR}/artifacts"
@@ -388,8 +390,9 @@
       # rebuilds keep working.
       docker run --rm -v php-portable-libdatadog-php:/vol alpine \
         rm -rf /vol/cargo-target
-      TERM=dumb ./gradlew test8.3-release-ssi test8.4-release-zts-ssi \
-        --info -Pbuildscan --scan -PcheckCoreDumps -PuseHelperRustCoverage
+      # Gradle forwards the appsec CMake output: a Depot 503 in its FetchContent exits 75, which is retried.
+      TERM=dumb ../../../.gitlab/run-with-retryable-download.sh ./gradlew test8.3-release-ssi test8.4-release-zts-ssi \
+        --info -Pbuildscan --scan -PcheckCoreDumps -PuseHelperRustCoverage || exit $?
       TERM=dumb ./gradlew saveCaches --info
     - |
       cd "$CI_PROJECT_DIR"
@@ -432,15 +435,15 @@
       sudo apt-get update && sudo apt-get install -y jq gcovr llvm-21 clang-21
     - cd appsec/build
     - |
-      cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_ENABLE_COVERAGE=ON \
+      ../../.gitlab/run-with-retryable-download.sh cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_ENABLE_COVERAGE=ON \
         -DDD_APPSEC_TESTING=ON -DCMAKE_CXX_FLAGS="-stdlib=libc++" \
         -DCMAKE_C_COMPILER=/usr/bin/clang-21 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-21 \
         -DCMAKE_CXX_LINK_FLAGS="-stdlib=libc++" \
-        -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache"
+        -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" || exit $?
     - |
       export PATH=$PATH:$HOME/.cargo/bin
       LLVM_PROFILE_FILE="/tmp/cov-ext/%p.profraw" \
-        VERBOSE=1 make -j 4 xtest
+        VERBOSE=1 ../../.gitlab/run-with-retryable-download.sh make -j 4 xtest || exit $?
     - |
       cd /tmp/cov-ext
       llvm-profdata-21 merge -sparse *.profraw -o default.profdata

.gitlab/generate-common.php

@@ -61,19 +61,24 @@ function windows_git_setup() {
     }
     Pop-Location
     $remaining = Get-ChildItem -Path . -Force -ErrorAction SilentlyContinue
-    if ($remaining) { Write-Host "WARNING: could not remove: $($remaining.Name -join ', ')" }
+    # git init would reuse leftovers (a clone used to refuse them): retry instead.
+    if ($remaining) { Write-Host "ERROR: could not remove: $($remaining.Name -join ', '); exiting 75 so GitLab auto-retries."; exit 75 }
     Write-Host "Cleanup complete."
 
     # PS 5.1 ignores $PSNativeCommandUseErrorActionPreference — use $LASTEXITCODE checks instead.
     $ErrorActionPreference = 'Stop'
 
-    # Manual git clone with proper config
-    Write-Host "Cloning repository..."
+    # Fetch the pipeline commit by SHA: once the branch moves on, a --branch clone no longer contains it.
+    Write-Host "Fetching $env:CI_COMMIT_SHA..."
     git config --global core.longpaths true
     git config --global core.symlinks true
-    git clone --branch $env:CI_COMMIT_REF_NAME $env:CI_REPOSITORY_URL .
+    git init -q .
+    if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+    git remote add origin $env:CI_REPOSITORY_URL
+    if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+    git fetch --depth 50 origin $env:CI_COMMIT_SHA
     if ($LASTEXITCODE -ne 0) {
-        Write-Host "ERROR: git clone failed (likely transient network); exiting 75 so GitLab auto-retries. Remaining workspace contents:"
+        Write-Host "ERROR: git fetch failed (likely transient network); exiting 75 so GitLab auto-retries. Remaining workspace contents:"
         Get-ChildItem -Force | Select-Object Name
         exit 75
     }
@@ -113,10 +118,11 @@ function windows_git_setup_with_packages() {
         return;
     }
 
+    // `|| exit $?` keeps git's 128 (retried) on the centos-7 images, where bash 4.2 turns an errexit status into 1.
     $step  = <<<STEP
     - git config --global --add safe.directory "\${CI_PROJECT_DIR}/libdatadog"
-    - (cd libdatadog; git fetch --depth=1 origin "$_libdatadog_sha" 2>/dev/null || git fetch origin "$_libdatadog_sha" 2>&1)
-    - (cd libdatadog; git checkout FETCH_HEAD)
+    - (cd libdatadog && { git fetch --depth=1 origin "$_libdatadog_sha" 2>/dev/null || git fetch origin "$_libdatadog_sha" 2>&1; }) || exit \$?
+    - (cd libdatadog && git checkout FETCH_HEAD) || exit \$?
 STEP;
 
     // Scripts whose presence in a job block means the job compiles Rust from libdatadog.

.gitlab/generate-package.php

@@ -182,12 +182,10 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
 # dd-trace-php release packaging
 "prepare code":
   stage: prepare
-  image: registry.ddbuild.io/images/mirror/composer:2
+  image: registry.ddbuild.io/images/mirror/composer:2.10.3@sha256:af98f42dfff7c68ba8d53c2164fd9fde1087b7d449514baa38c418b1f6bc4bac
   tags: [ "arch:amd64" ]
   script:
     - ./.gitlab/append-build-id.sh
-    # Upgrading composer
-    - composer self-update --no-interaction
     # Installing dependencies with composer
     - |
       export COMPOSER_MEMORY_LIMIT=-1 # disable composer memory limit completely
@@ -1125,7 +1123,8 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
     DD_TRACE_AGENT_PORT: 80
     DD_TRACE_AGENT_FLUSH_INTERVAL: 1000
   script:
-    - ./dockerfiles/verify_packages/verify.sh
+    # `|| exit $?` keeps install.sh's exit 75 on centos:7, where bash 4.2 turns an errexit status into 1.
+    - ./dockerfiles/verify_packages/verify.sh || exit $?
 
 
 "verify alpine":
@@ -1244,7 +1243,8 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
                -e 's|http://deb.debian.org/debian|http://snapshot.debian.org/archive/debian/20260901T000000Z|g' /etc/apt/sources.list
         echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99no-check-valid-until
       fi
-    - apt-get update || exit 75
+    # Bound apt: a stalled deb.debian.org fetch otherwise hangs until the 1 h job timeout; 75 is retried.
+    - timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 update || exit 75
     - |
       # apt-get update still exits 0 when the rewrite silently no-ops, so assert on the URIs apt would actually fetch from.
       if [ "$(. /etc/os-release; echo $VERSION_CODENAME)" = "bullseye" ]; then
@@ -1255,7 +1255,7 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
         if [ -z "$uris" ]; then echo "FAIL: could not resolve any apt URIs"; exit 1; fi
         if [ -n "$bad" ]; then echo "FAIL: bullseye apt sources not pinned; apt would still fetch from: $bad"; exit 1; fi
       fi
-    - apt-get install -y curl || exit 75
+    - timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 install -y curl || exit 75
 
 # Thread-mode sidecar under a privilege-dropping PHP-FPM: root master, www-data pool workers.
 # One combination only - this verifies runtime behaviour of the sidecar's privilege drop, not

.gitlab/generate-tracer.php

@@ -297,15 +297,14 @@ function windows_test_c_job($job_name, $thread_safety, $targets) {
 
 "Prepare code":
   stage: compile
-  image: registry.ddbuild.io/images/mirror/php:8.2-cli
+  # Same image as the package "prepare code": composer and unzip are baked in, so no apt or getcomposer.org fetch.
+  image: registry.ddbuild.io/images/mirror/composer:2.10.3@sha256:af98f42dfff7c68ba8d53c2164fd9fde1087b7d449514baa38c418b1f6bc4bac
   tags: [ "arch:amd64" ]
   needs: []
   variables:
     KUBERNETES_CPU_REQUEST: 1
     KUBERNETES_MEMORY_REQUEST: 2Gi
   before_script:
-    - apt update && apt install -y unzip
-    - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php && mv composer.phar /usr/local/bin/composer
     - composer update --no-interaction
   script:
     - make generate
@@ -843,6 +842,13 @@ function windows_test_c_job($job_name, $thread_safety, $targets) {
     KUBERNETES_MEMORY_REQUEST: 6Gi
     KUBERNETES_MEMORY_LIMIT: 6Gi
 <?php endif; ?>
+<?php if ($target === "test_integrations_frankenphp"): ?>
+  script:
+    # Build FrankenPHP as its own step, in the proxy-free env the harness gave it: from inside PHPUnit its exit 75 is lost.
+    - unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy
+    - tooling/bin/install-frankenphp.sh || exit $?
+    - !reference [.cli_integration_test, script]
+<?php endif; ?>
 
 <?php
             endforeach;

dockerfiles/verify_packages/debian/install.sh

@@ -19,9 +19,12 @@ retry_or_tempfail() {
     done
 }
 
+# Bound apt: a stalled mirror fetch otherwise hangs until the 1 h job timeout.
+apt_get() { timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 "$@"; }
+
 # Common installations
-retry_or_tempfail apt-get update
-retry_or_tempfail apt-get install -y \
+retry_or_tempfail apt_get update
+retry_or_tempfail apt_get install -y \
     apt-transport-https \
     lsb-release \
     ca-certificates \
@@ -37,20 +40,20 @@ sed -i 's/*:80/127.0.0.1:8081/' /etc/apache2/sites-enabled/000-default.conf
 
 # Installing php
 if [ "${INSTALL_MODE}" = "native" ]; then
-    retry_or_tempfail apt-get install -y ${PHP_PACKAGE}
+    retry_or_tempfail apt_get install -y ${PHP_PACKAGE}
 elif [ "${INSTALL_MODE}" = "sury" ]; then
     retry_or_tempfail curl -sSfL -o /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg
     sh -c 'echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list'
-    retry_or_tempfail apt-get update
-    retry_or_tempfail apt-get install -y \
+    retry_or_tempfail apt_get update
+    retry_or_tempfail apt_get install -y \
         php${PHP_VERSION}-cli \
         php${PHP_VERSION}-curl \
         php${PHP_VERSION}-fpm \
         libapache2-mod-php${PHP_VERSION}
         WWW_CONF=/etc/php/${PHP_VERSION}/fpm/pool.d/www.conf
         PHP_FPM_BIN=php-fpm${PHP_VERSION}
     if [ "${PHP_VERSION}" != "8.5" ]; then
-        retry_or_tempfail apt-get install -y \
+        retry_or_tempfail apt_get install -y \
             php${PHP_VERSION}-opcache
     fi
 else

dockerfiles/verify_packages/tar_gz/install.sh

@@ -19,9 +19,12 @@ retry_or_tempfail() {
     done
 }
 
+# Bound apt: a stalled mirror fetch otherwise hangs until the 1 h job timeout.
+apt_get() { timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 "$@"; }
+
 # Common installations
-retry_or_tempfail apt-get update
-retry_or_tempfail apt-get install -y \
+retry_or_tempfail apt_get update
+retry_or_tempfail apt_get install -y \
     apt-transport-https \
     lsb-release \
     ca-certificates \
@@ -33,8 +36,8 @@ retry_or_tempfail apt-get install -y \
 
 retry_or_tempfail curl -sSfL -o /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg
 sh -c 'echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list'
-retry_or_tempfail apt-get update
-retry_or_tempfail apt-get install -y \
+retry_or_tempfail apt_get update
+retry_or_tempfail apt_get install -y \
     php${PHP_VERSION}-cli \
     php${PHP_VERSION}-curl \
     php${PHP_VERSION}-fpm \

dockerfiles/verify_packages/verify_windows.ps1

@@ -13,6 +13,8 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 choco install -y 7zip
 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 refreshenv
+# choco exits 0 after a feed error with "installed 0/0 packages"; treat a missing php as infra (75 is retried).
+if (-not (Get-Command php -ErrorAction SilentlyContinue)) { Write-Host "php is not installed after choco install (feed failure?); exiting 75 so GitLab auto-retries"; exit 75 }
 
 php build/packages/datadog-setup.php --php-bin=all --file=$(ls build/packages/dd-library-php-*-x86_64-windows.tar.gz)
 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

tests/Sapi/OctaneServer/OctaneServer.php

@@ -71,8 +71,10 @@ public function start()
             $this->envs["DD_TRACE_AGENT_TEST_SESSION_TOKEN"] = $token;
         }
 
+        // Pin the pool like the Swoole and RoadRunner harnesses: Octane's "auto" boots 2 x node vCPUs
+        // Laravel workers (256 on a 128-vCPU CI node), which OOMs the job.
         $cmd = sprintf(
-            PHP_BINARY . ' %s %s octane:start --server=swoole --host=%s --port=%d',
+            PHP_BINARY . ' %s %s octane:start --server=swoole --host=%s --port=%d --workers=2 --task-workers=1',
             new IniSerializer($this->inis),
             $this->artisanFile,
             $this->host,

tooling/bin/install-frankenphp.sh

@@ -14,7 +14,8 @@ FRANKENPHP_SRC_DIR=/usr/local/src/frankenphp
 rm -rf $FRANKENPHP_SRC_DIR
 mkdir -p $FRANKENPHP_SRC_DIR
 
-curl -fLo /tmp/frankenphp.tar.gz ${frankenphpTarGzUrl}
+# Retries transient failures (Depot 5xx) and then exits 75, which GitLab retries.
+"$(dirname "${BASH_SOURCE[0]}")/../../.gitlab/download-file.sh" "${frankenphpTarGzUrl}" /tmp/frankenphp.tar.gz
 tar xf /tmp/frankenphp.tar.gz -C ${FRANKENPHP_SRC_DIR} --strip-components=1
 rm -f /tmp/frankenphp.tar.gz