From 76985d4685df6fa290d932a18ace8488118ca5bb Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Fri, 16 Oct 2026 23:37:57 +0200 Subject: [PATCH 1/9] ci: fetch the pipeline commit by SHA in Windows jobs A --branch clone misses the pipeline SHA once the branch moves on, and its exit 128 was retried twice for nothing. GitLab serves any SHA (Gitaly sets uploadpack.allowAnySHA1InWant), so fetch the pipeline commit directly. --- .gitlab/generate-common.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.gitlab/generate-common.php b/.gitlab/generate-common.php index 932fcd84a9d..832e2b373d1 100644 --- a/.gitlab/generate-common.php +++ b/.gitlab/generate-common.php @@ -61,19 +61,24 @@ function windows_git_setup() { } Pop-Location $remaining = Get-ChildItem -Path . -Force -ErrorAction SilentlyContinue - if ($remaining) { Write-Host "WARNING: could not remove: $($remaining.Name -join ', ')" } + # git init would reuse leftovers (a clone used to refuse them): retry instead. + if ($remaining) { Write-Host "ERROR: could not remove: $($remaining.Name -join ', '); exiting 75 so GitLab auto-retries."; exit 75 } Write-Host "Cleanup complete." # PS 5.1 ignores $PSNativeCommandUseErrorActionPreference — use $LASTEXITCODE checks instead. $ErrorActionPreference = 'Stop' - # Manual git clone with proper config - Write-Host "Cloning repository..." + # Fetch the pipeline commit by SHA: once the branch moves on, a --branch clone no longer contains it. + Write-Host "Fetching $env:CI_COMMIT_SHA..." git config --global core.longpaths true git config --global core.symlinks true - git clone --branch $env:CI_COMMIT_REF_NAME $env:CI_REPOSITORY_URL . + git init -q . + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + git remote add origin $env:CI_REPOSITORY_URL + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + git fetch --depth 50 origin $env:CI_COMMIT_SHA if ($LASTEXITCODE -ne 0) { - Write-Host "ERROR: git clone failed (likely transient network); exiting 75 so GitLab auto-retries. Remaining workspace contents:" + Write-Host "ERROR: git fetch failed (likely transient network); exiting 75 so GitLab auto-retries. Remaining workspace contents:" Get-ChildItem -Force | Select-Object Name exit 75 } From 2d8ce6c2188cc9e98242bb52d8f76949cb0f0b35 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Fri, 16 Oct 2026 23:38:06 +0200 Subject: [PATCH 2/9] ci: run tracer Prepare code on the composer mirror image The package "prepare code" already uses it; it drops the apt update and getcomposer.org fetch. An apt hang there delayed every tracer test job by 59 min (pipeline 144816042). --- .gitlab/generate-tracer.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/generate-tracer.php b/.gitlab/generate-tracer.php index c74064c827a..d01257dddf9 100644 --- a/.gitlab/generate-tracer.php +++ b/.gitlab/generate-tracer.php @@ -297,15 +297,14 @@ function windows_test_c_job($job_name, $thread_safety, $targets) { "Prepare code": stage: compile - image: registry.ddbuild.io/images/mirror/php:8.2-cli + # Same image as the package "prepare code": composer and unzip are baked in, so no apt or getcomposer.org fetch. + image: registry.ddbuild.io/images/mirror/composer:2 tags: [ "arch:amd64" ] needs: [] variables: KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_REQUEST: 2Gi before_script: - - apt update && apt install -y unzip - - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php && mv composer.phar /usr/local/bin/composer - composer update --no-interaction script: - make generate From ddc11decc092b77fa710d7d685eac0fb335a612a Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Fri, 16 Oct 2026 23:43:49 +0200 Subject: [PATCH 3/9] ci: bound apt in the verify debian and .tar.gz jobs A stalled deb.debian.org fetch hung apt-get update until the 1 h job timeout (4 jobs, 60 min each). Cap each apt-get call at 300 s with a 30 s HTTP timeout; the existing exit 75 paths then retry the job. --- .gitlab/generate-package.php | 5 +++-- dockerfiles/verify_packages/debian/install.sh | 15 +++++++++------ dockerfiles/verify_packages/tar_gz/install.sh | 11 +++++++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index d31b9d849f4..996e77361dc 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1244,7 +1244,8 @@ function package_extension_jobs(array $platform, string $kind = "all"): array -e 's|http://deb.debian.org/debian|http://snapshot.debian.org/archive/debian/20260901T000000Z|g' /etc/apt/sources.list echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99no-check-valid-until fi - - apt-get update || exit 75 + # Bound apt: a stalled deb.debian.org fetch otherwise hangs until the 1 h job timeout; 75 is retried. + - timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 update || exit 75 - | # apt-get update still exits 0 when the rewrite silently no-ops, so assert on the URIs apt would actually fetch from. if [ "$(. /etc/os-release; echo $VERSION_CODENAME)" = "bullseye" ]; then @@ -1255,7 +1256,7 @@ function package_extension_jobs(array $platform, string $kind = "all"): array if [ -z "$uris" ]; then echo "FAIL: could not resolve any apt URIs"; exit 1; fi if [ -n "$bad" ]; then echo "FAIL: bullseye apt sources not pinned; apt would still fetch from: $bad"; exit 1; fi fi - - apt-get install -y curl || exit 75 + - timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 install -y curl || exit 75 # Thread-mode sidecar under a privilege-dropping PHP-FPM: root master, www-data pool workers. # One combination only - this verifies runtime behaviour of the sidecar's privilege drop, not diff --git a/dockerfiles/verify_packages/debian/install.sh b/dockerfiles/verify_packages/debian/install.sh index d9bc669dfde..eb143dceb6e 100644 --- a/dockerfiles/verify_packages/debian/install.sh +++ b/dockerfiles/verify_packages/debian/install.sh @@ -19,9 +19,12 @@ retry_or_tempfail() { done } +# Bound apt: a stalled mirror fetch otherwise hangs until the 1 h job timeout. +apt_get() { timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 "$@"; } + # Common installations -retry_or_tempfail apt-get update -retry_or_tempfail apt-get install -y \ +retry_or_tempfail apt_get update +retry_or_tempfail apt_get install -y \ apt-transport-https \ lsb-release \ ca-certificates \ @@ -37,12 +40,12 @@ sed -i 's/*:80/127.0.0.1:8081/' /etc/apache2/sites-enabled/000-default.conf # Installing php if [ "${INSTALL_MODE}" = "native" ]; then - retry_or_tempfail apt-get install -y ${PHP_PACKAGE} + retry_or_tempfail apt_get install -y ${PHP_PACKAGE} elif [ "${INSTALL_MODE}" = "sury" ]; then retry_or_tempfail curl -sSfL -o /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg sh -c 'echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list' - retry_or_tempfail apt-get update - retry_or_tempfail apt-get install -y \ + retry_or_tempfail apt_get update + retry_or_tempfail apt_get install -y \ php${PHP_VERSION}-cli \ php${PHP_VERSION}-curl \ php${PHP_VERSION}-fpm \ @@ -50,7 +53,7 @@ elif [ "${INSTALL_MODE}" = "sury" ]; then WWW_CONF=/etc/php/${PHP_VERSION}/fpm/pool.d/www.conf PHP_FPM_BIN=php-fpm${PHP_VERSION} if [ "${PHP_VERSION}" != "8.5" ]; then - retry_or_tempfail apt-get install -y \ + retry_or_tempfail apt_get install -y \ php${PHP_VERSION}-opcache fi else diff --git a/dockerfiles/verify_packages/tar_gz/install.sh b/dockerfiles/verify_packages/tar_gz/install.sh index dbfdf48aa0b..525b888847f 100755 --- a/dockerfiles/verify_packages/tar_gz/install.sh +++ b/dockerfiles/verify_packages/tar_gz/install.sh @@ -19,9 +19,12 @@ retry_or_tempfail() { done } +# Bound apt: a stalled mirror fetch otherwise hangs until the 1 h job timeout. +apt_get() { timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 "$@"; } + # Common installations -retry_or_tempfail apt-get update -retry_or_tempfail apt-get install -y \ +retry_or_tempfail apt_get update +retry_or_tempfail apt_get install -y \ apt-transport-https \ lsb-release \ ca-certificates \ @@ -33,8 +36,8 @@ retry_or_tempfail apt-get install -y \ retry_or_tempfail curl -sSfL -o /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg sh -c 'echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list' -retry_or_tempfail apt-get update -retry_or_tempfail apt-get install -y \ +retry_or_tempfail apt_get update +retry_or_tempfail apt_get install -y \ php${PHP_VERSION}-cli \ php${PHP_VERSION}-curl \ php${PHP_VERSION}-fpm \ From 27c96d8182ac49fcb81a2d4df1798ac24b69392a Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Fri, 16 Oct 2026 23:46:14 +0200 Subject: [PATCH 4/9] test: pin the Octane test server's worker count Octane's "auto" starts one worker and one task worker per node vCPU, so 128-vCPU CI nodes boot 256 Laravel workers and OOM the 6Gi job (6/6 there, 981/981 pass at <=96). Use 2 workers and 1 task worker, like the Swoole and RoadRunner harnesses. --- tests/Sapi/OctaneServer/OctaneServer.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Sapi/OctaneServer/OctaneServer.php b/tests/Sapi/OctaneServer/OctaneServer.php index 55ed8da2215..693148681e9 100644 --- a/tests/Sapi/OctaneServer/OctaneServer.php +++ b/tests/Sapi/OctaneServer/OctaneServer.php @@ -71,8 +71,10 @@ public function start() $this->envs["DD_TRACE_AGENT_TEST_SESSION_TOKEN"] = $token; } + // Pin the pool like the Swoole and RoadRunner harnesses: Octane's "auto" boots 2 x node vCPUs + // Laravel workers (256 on a 128-vCPU CI node), which OOMs the job. $cmd = sprintf( - PHP_BINARY . ' %s %s octane:start --server=swoole --host=%s --port=%d', + PHP_BINARY . ' %s %s octane:start --server=swoole --host=%s --port=%d --workers=2 --task-workers=1', new IniSerializer($this->inis), $this->artisanFile, $this->host, From ae37d2595c77d3c9112f873fe4012c06c855f726 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Fri, 16 Oct 2026 23:48:56 +0200 Subject: [PATCH 5/9] ci: retry Depot download failures in appsec tests and FrankenPHP builds Run the appsec test cmake and make xtest through run-with-retryable-download.sh, as build-appsec.sh does. Fetch the FrankenPHP tarball with download-file.sh and build it as its own script line, so its exit 75 reaches GitLab. --- .gitlab/generate-appsec.php | 13 +++++++------ .gitlab/generate-tracer.php | 7 +++++++ tooling/bin/install-frankenphp.sh | 3 ++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.gitlab/generate-appsec.php b/.gitlab/generate-appsec.php index 7e3156181e6..18216461f28 100644 --- a/.gitlab/generate-appsec.php +++ b/.gitlab/generate-appsec.php @@ -160,11 +160,12 @@ - switch-php $SWITCH_PHP_VERSION - cd appsec/build - if [[ "$SWITCH_PHP_VERSION" == *"asan"* ]]; then ASAN_FLAG=ON; else ASAN_FLAG=OFF; fi - - "cmake .. -DCMAKE_BUILD_TYPE=Debug + # FetchContent downloads boost and rapidjson at configure time: a Depot 503 or a boost transfer error there exits 75 and is retried. + - "../../.gitlab/run-with-retryable-download.sh cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS='-stdlib=libc++' -DCMAKE_CXX_LINK_FLAGS='-stdlib=libc++' -DDD_APPSEC_TESTING=ON -DBOOST_CACHE_PREFIX=$CI_PROJECT_DIR/boost-cache - -DENABLE_ASAN=$ASAN_FLAG" - - ASAN_OPTIONS=malloc_context_size=0 make -j 4 xtest + -DENABLE_ASAN=$ASAN_FLAG || exit $?" + - ASAN_OPTIONS=malloc_context_size=0 ../../.gitlab/run-with-retryable-download.sh make -j 4 xtest || exit $? .appsec_integration_tests: stage: test @@ -432,15 +433,15 @@ sudo apt-get update && sudo apt-get install -y jq gcovr llvm-21 clang-21 - cd appsec/build - | - cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_ENABLE_COVERAGE=ON \ + ../../.gitlab/run-with-retryable-download.sh cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_ENABLE_COVERAGE=ON \ -DDD_APPSEC_TESTING=ON -DCMAKE_CXX_FLAGS="-stdlib=libc++" \ -DCMAKE_C_COMPILER=/usr/bin/clang-21 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-21 \ -DCMAKE_CXX_LINK_FLAGS="-stdlib=libc++" \ - -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" + -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" || exit $? - | export PATH=$PATH:$HOME/.cargo/bin LLVM_PROFILE_FILE="/tmp/cov-ext/%p.profraw" \ - VERBOSE=1 make -j 4 xtest + VERBOSE=1 ../../.gitlab/run-with-retryable-download.sh make -j 4 xtest || exit $? - | cd /tmp/cov-ext llvm-profdata-21 merge -sparse *.profraw -o default.profdata diff --git a/.gitlab/generate-tracer.php b/.gitlab/generate-tracer.php index d01257dddf9..b2dde569171 100644 --- a/.gitlab/generate-tracer.php +++ b/.gitlab/generate-tracer.php @@ -842,6 +842,13 @@ function windows_test_c_job($job_name, $thread_safety, $targets) { KUBERNETES_MEMORY_REQUEST: 6Gi KUBERNETES_MEMORY_LIMIT: 6Gi + + script: + # Build FrankenPHP as its own step, in the proxy-free env the harness gave it: from inside PHPUnit its exit 75 is lost. + - unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy + - tooling/bin/install-frankenphp.sh || exit $? + - !reference [.cli_integration_test, script] + Date: Fri, 16 Oct 2026 23:49:41 +0200 Subject: [PATCH 6/9] ci: retry verify windows when chocolatey installs no php choco exits 0 after a feed error ("installed 0/0 packages"), and the job then failed with exit 1 on the missing php (job 2129816747). Exit 75 instead, which the default retry covers. --- dockerfiles/verify_packages/verify_windows.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dockerfiles/verify_packages/verify_windows.ps1 b/dockerfiles/verify_packages/verify_windows.ps1 index d09276413eb..a333652df54 100644 --- a/dockerfiles/verify_packages/verify_windows.ps1 +++ b/dockerfiles/verify_packages/verify_windows.ps1 @@ -13,6 +13,8 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } choco install -y 7zip if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } refreshenv +# choco exits 0 after a feed error with "installed 0/0 packages"; treat a missing php as infra (75 is retried). +if (-not (Get-Command php -ErrorAction SilentlyContinue)) { Write-Host "php is not installed after choco install (feed failure?); exiting 75 so GitLab auto-retries"; exit 75 } php build/packages/datadog-setup.php --php-bin=all --file=$(ls build/packages/dd-library-php-*-x86_64-windows.tar.gz) if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } From d0dc93fe29087b253ee0c86178d44b4ab5bcccfe Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Fri, 16 Oct 2026 23:49:55 +0200 Subject: [PATCH 7/9] ci: retry Depot 503s in the appsec gradle test jobs The gradle builds run the appsec CMake FetchContent outside build-appsec.sh, so a Depot 503 failed with exit 1 (job 2137703728). Run the gradle test calls of .appsec_integration_tests and helper-rust integration coverage through run-with-retryable-download.sh. --- .gitlab/generate-appsec.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab/generate-appsec.php b/.gitlab/generate-appsec.php index 18216461f28..940e991455c 100644 --- a/.gitlab/generate-appsec.php +++ b/.gitlab/generate-appsec.php @@ -194,7 +194,8 @@ TERM=dumb ./gradlew loadCaches --info fi - TERM=dumb ./gradlew $targets --info -Pbuildscan --scan -PcheckCoreDumps + # Gradle forwards the appsec CMake output: a Depot 503 in its FetchContent exits 75, which is retried. + TERM=dumb ../../../.gitlab/run-with-retryable-download.sh ./gradlew $targets --info -Pbuildscan --scan -PcheckCoreDumps || exit $? TERM=dumb ./gradlew saveCaches --info after_script: - mkdir -p "${CI_PROJECT_DIR}/artifacts" @@ -389,8 +390,9 @@ # rebuilds keep working. docker run --rm -v php-portable-libdatadog-php:/vol alpine \ rm -rf /vol/cargo-target - TERM=dumb ./gradlew test8.3-release-ssi test8.4-release-zts-ssi \ - --info -Pbuildscan --scan -PcheckCoreDumps -PuseHelperRustCoverage + # Gradle forwards the appsec CMake output: a Depot 503 in its FetchContent exits 75, which is retried. + TERM=dumb ../../../.gitlab/run-with-retryable-download.sh ./gradlew test8.3-release-ssi test8.4-release-zts-ssi \ + --info -Pbuildscan --scan -PcheckCoreDumps -PuseHelperRustCoverage || exit $? TERM=dumb ./gradlew saveCaches --info - | cd "$CI_PROJECT_DIR" From 6b03f4c965a5ee5aa14df8c69ea5965f6cb56c56 Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Fri, 16 Oct 2026 23:50:52 +0200 Subject: [PATCH 8/9] ci: keep retryable exit codes on the centos-7 images bash 4.2 turns any errexit status inside the runner's eval into 1, so verify centos lost install.sh's exit 75. End that line, and the libdatadog-override git steps (128), with `|| exit $?`, as #4318 did for compile appsec. --- .gitlab/generate-common.php | 5 +++-- .gitlab/generate-package.php | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab/generate-common.php b/.gitlab/generate-common.php index 832e2b373d1..9463fecd18a 100644 --- a/.gitlab/generate-common.php +++ b/.gitlab/generate-common.php @@ -118,10 +118,11 @@ function windows_git_setup_with_packages() { return; } + // `|| exit $?` keeps git's 128 (retried) on the centos-7 images, where bash 4.2 turns an errexit status into 1. $step = <</dev/null || git fetch origin "$_libdatadog_sha" 2>&1) - - (cd libdatadog; git checkout FETCH_HEAD) + - (cd libdatadog && { git fetch --depth=1 origin "$_libdatadog_sha" 2>/dev/null || git fetch origin "$_libdatadog_sha" 2>&1; }) || exit \$? + - (cd libdatadog && git checkout FETCH_HEAD) || exit \$? STEP; // Scripts whose presence in a job block means the job compiles Rust from libdatadog. diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 996e77361dc..0fea060e0ff 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1125,7 +1125,8 @@ function package_extension_jobs(array $platform, string $kind = "all"): array DD_TRACE_AGENT_PORT: 80 DD_TRACE_AGENT_FLUSH_INTERVAL: 1000 script: - - ./dockerfiles/verify_packages/verify.sh + # `|| exit $?` keeps install.sh's exit 75 on centos:7, where bash 4.2 turns an errexit status into 1. + - ./dockerfiles/verify_packages/verify.sh || exit $? "verify alpine": From 1b0ddcf8c0136d1d34aa3eb7c89cfd68772f982a Mon Sep 17 00:00:00 2001 From: Alexandre Rulleau Date: Sat, 17 Oct 2026 01:31:11 +0200 Subject: [PATCH 9/9] ci: use the Composer 2.10.3 mirror image in Prepare code The mirror's composer:2 is Composer 2.5.1, which stalls behind the proxy and never retries curl error 28 (job 2148933797: 300 s). Both prepare code jobs now use composer:2.10.3 pinned by digest, so the package job no longer self-updates; make generate output is byte-identical. --- .gitlab/generate-package.php | 4 +--- .gitlab/generate-tracer.php | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 0fea060e0ff..a835e3c7dd5 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -182,12 +182,10 @@ function package_extension_jobs(array $platform, string $kind = "all"): array # dd-trace-php release packaging "prepare code": stage: prepare - image: registry.ddbuild.io/images/mirror/composer:2 + image: registry.ddbuild.io/images/mirror/composer:2.10.3@sha256:af98f42dfff7c68ba8d53c2164fd9fde1087b7d449514baa38c418b1f6bc4bac tags: [ "arch:amd64" ] script: - ./.gitlab/append-build-id.sh - # Upgrading composer - - composer self-update --no-interaction # Installing dependencies with composer - | export COMPOSER_MEMORY_LIMIT=-1 # disable composer memory limit completely diff --git a/.gitlab/generate-tracer.php b/.gitlab/generate-tracer.php index b2dde569171..1b847966bad 100644 --- a/.gitlab/generate-tracer.php +++ b/.gitlab/generate-tracer.php @@ -298,7 +298,7 @@ function windows_test_c_job($job_name, $thread_safety, $targets) { "Prepare code": stage: compile # Same image as the package "prepare code": composer and unzip are baked in, so no apt or getcomposer.org fetch. - image: registry.ddbuild.io/images/mirror/composer:2 + image: registry.ddbuild.io/images/mirror/composer:2.10.3@sha256:af98f42dfff7c68ba8d53c2164fd9fde1087b7d449514baa38c418b1f6bc4bac tags: [ "arch:amd64" ] needs: [] variables: