diff --git a/.gitlab/generate-appsec.php b/.gitlab/generate-appsec.php index 7e3156181e6..940e991455c 100644 --- a/.gitlab/generate-appsec.php +++ b/.gitlab/generate-appsec.php @@ -160,11 +160,12 @@ - switch-php $SWITCH_PHP_VERSION - cd appsec/build - if [[ "$SWITCH_PHP_VERSION" == *"asan"* ]]; then ASAN_FLAG=ON; else ASAN_FLAG=OFF; fi - - "cmake .. -DCMAKE_BUILD_TYPE=Debug + # FetchContent downloads boost and rapidjson at configure time: a Depot 503 or a boost transfer error there exits 75 and is retried. + - "../../.gitlab/run-with-retryable-download.sh cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS='-stdlib=libc++' -DCMAKE_CXX_LINK_FLAGS='-stdlib=libc++' -DDD_APPSEC_TESTING=ON -DBOOST_CACHE_PREFIX=$CI_PROJECT_DIR/boost-cache - -DENABLE_ASAN=$ASAN_FLAG" - - ASAN_OPTIONS=malloc_context_size=0 make -j 4 xtest + -DENABLE_ASAN=$ASAN_FLAG || exit $?" + - ASAN_OPTIONS=malloc_context_size=0 ../../.gitlab/run-with-retryable-download.sh make -j 4 xtest || exit $? .appsec_integration_tests: stage: test @@ -193,7 +194,8 @@ TERM=dumb ./gradlew loadCaches --info fi - TERM=dumb ./gradlew $targets --info -Pbuildscan --scan -PcheckCoreDumps + # Gradle forwards the appsec CMake output: a Depot 503 in its FetchContent exits 75, which is retried. + TERM=dumb ../../../.gitlab/run-with-retryable-download.sh ./gradlew $targets --info -Pbuildscan --scan -PcheckCoreDumps || exit $? TERM=dumb ./gradlew saveCaches --info after_script: - mkdir -p "${CI_PROJECT_DIR}/artifacts" @@ -388,8 +390,9 @@ # rebuilds keep working. docker run --rm -v php-portable-libdatadog-php:/vol alpine \ rm -rf /vol/cargo-target - TERM=dumb ./gradlew test8.3-release-ssi test8.4-release-zts-ssi \ - --info -Pbuildscan --scan -PcheckCoreDumps -PuseHelperRustCoverage + # Gradle forwards the appsec CMake output: a Depot 503 in its FetchContent exits 75, which is retried. + TERM=dumb ../../../.gitlab/run-with-retryable-download.sh ./gradlew test8.3-release-ssi test8.4-release-zts-ssi \ + --info -Pbuildscan --scan -PcheckCoreDumps -PuseHelperRustCoverage || exit $? TERM=dumb ./gradlew saveCaches --info - | cd "$CI_PROJECT_DIR" @@ -432,15 +435,15 @@ sudo apt-get update && sudo apt-get install -y jq gcovr llvm-21 clang-21 - cd appsec/build - | - cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_ENABLE_COVERAGE=ON \ + ../../.gitlab/run-with-retryable-download.sh cmake .. -DCMAKE_BUILD_TYPE=Debug -DDD_APPSEC_ENABLE_COVERAGE=ON \ -DDD_APPSEC_TESTING=ON -DCMAKE_CXX_FLAGS="-stdlib=libc++" \ -DCMAKE_C_COMPILER=/usr/bin/clang-21 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-21 \ -DCMAKE_CXX_LINK_FLAGS="-stdlib=libc++" \ - -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" + -DBOOST_CACHE_PREFIX="$CI_PROJECT_DIR/boost-cache" || exit $? - | export PATH=$PATH:$HOME/.cargo/bin LLVM_PROFILE_FILE="/tmp/cov-ext/%p.profraw" \ - VERBOSE=1 make -j 4 xtest + VERBOSE=1 ../../.gitlab/run-with-retryable-download.sh make -j 4 xtest || exit $? - | cd /tmp/cov-ext llvm-profdata-21 merge -sparse *.profraw -o default.profdata diff --git a/.gitlab/generate-common.php b/.gitlab/generate-common.php index 932fcd84a9d..9463fecd18a 100644 --- a/.gitlab/generate-common.php +++ b/.gitlab/generate-common.php @@ -61,19 +61,24 @@ function windows_git_setup() { } Pop-Location $remaining = Get-ChildItem -Path . -Force -ErrorAction SilentlyContinue - if ($remaining) { Write-Host "WARNING: could not remove: $($remaining.Name -join ', ')" } + # git init would reuse leftovers (a clone used to refuse them): retry instead. + if ($remaining) { Write-Host "ERROR: could not remove: $($remaining.Name -join ', '); exiting 75 so GitLab auto-retries."; exit 75 } Write-Host "Cleanup complete." # PS 5.1 ignores $PSNativeCommandUseErrorActionPreference — use $LASTEXITCODE checks instead. $ErrorActionPreference = 'Stop' - # Manual git clone with proper config - Write-Host "Cloning repository..." + # Fetch the pipeline commit by SHA: once the branch moves on, a --branch clone no longer contains it. + Write-Host "Fetching $env:CI_COMMIT_SHA..." git config --global core.longpaths true git config --global core.symlinks true - git clone --branch $env:CI_COMMIT_REF_NAME $env:CI_REPOSITORY_URL . + git init -q . + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + git remote add origin $env:CI_REPOSITORY_URL + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + git fetch --depth 50 origin $env:CI_COMMIT_SHA if ($LASTEXITCODE -ne 0) { - Write-Host "ERROR: git clone failed (likely transient network); exiting 75 so GitLab auto-retries. Remaining workspace contents:" + Write-Host "ERROR: git fetch failed (likely transient network); exiting 75 so GitLab auto-retries. Remaining workspace contents:" Get-ChildItem -Force | Select-Object Name exit 75 } @@ -113,10 +118,11 @@ function windows_git_setup_with_packages() { return; } + // `|| exit $?` keeps git's 128 (retried) on the centos-7 images, where bash 4.2 turns an errexit status into 1. $step = <</dev/null || git fetch origin "$_libdatadog_sha" 2>&1) - - (cd libdatadog; git checkout FETCH_HEAD) + - (cd libdatadog && { git fetch --depth=1 origin "$_libdatadog_sha" 2>/dev/null || git fetch origin "$_libdatadog_sha" 2>&1; }) || exit \$? + - (cd libdatadog && git checkout FETCH_HEAD) || exit \$? STEP; // Scripts whose presence in a job block means the job compiles Rust from libdatadog. diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index d31b9d849f4..a835e3c7dd5 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -182,12 +182,10 @@ function package_extension_jobs(array $platform, string $kind = "all"): array # dd-trace-php release packaging "prepare code": stage: prepare - image: registry.ddbuild.io/images/mirror/composer:2 + image: registry.ddbuild.io/images/mirror/composer:2.10.3@sha256:af98f42dfff7c68ba8d53c2164fd9fde1087b7d449514baa38c418b1f6bc4bac tags: [ "arch:amd64" ] script: - ./.gitlab/append-build-id.sh - # Upgrading composer - - composer self-update --no-interaction # Installing dependencies with composer - | export COMPOSER_MEMORY_LIMIT=-1 # disable composer memory limit completely @@ -1125,7 +1123,8 @@ function package_extension_jobs(array $platform, string $kind = "all"): array DD_TRACE_AGENT_PORT: 80 DD_TRACE_AGENT_FLUSH_INTERVAL: 1000 script: - - ./dockerfiles/verify_packages/verify.sh + # `|| exit $?` keeps install.sh's exit 75 on centos:7, where bash 4.2 turns an errexit status into 1. + - ./dockerfiles/verify_packages/verify.sh || exit $? "verify alpine": @@ -1244,7 +1243,8 @@ function package_extension_jobs(array $platform, string $kind = "all"): array -e 's|http://deb.debian.org/debian|http://snapshot.debian.org/archive/debian/20260901T000000Z|g' /etc/apt/sources.list echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99no-check-valid-until fi - - apt-get update || exit 75 + # Bound apt: a stalled deb.debian.org fetch otherwise hangs until the 1 h job timeout; 75 is retried. + - timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 update || exit 75 - | # apt-get update still exits 0 when the rewrite silently no-ops, so assert on the URIs apt would actually fetch from. if [ "$(. /etc/os-release; echo $VERSION_CODENAME)" = "bullseye" ]; then @@ -1255,7 +1255,7 @@ function package_extension_jobs(array $platform, string $kind = "all"): array if [ -z "$uris" ]; then echo "FAIL: could not resolve any apt URIs"; exit 1; fi if [ -n "$bad" ]; then echo "FAIL: bullseye apt sources not pinned; apt would still fetch from: $bad"; exit 1; fi fi - - apt-get install -y curl || exit 75 + - timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 install -y curl || exit 75 # Thread-mode sidecar under a privilege-dropping PHP-FPM: root master, www-data pool workers. # One combination only - this verifies runtime behaviour of the sidecar's privilege drop, not diff --git a/.gitlab/generate-tracer.php b/.gitlab/generate-tracer.php index c74064c827a..1b847966bad 100644 --- a/.gitlab/generate-tracer.php +++ b/.gitlab/generate-tracer.php @@ -297,15 +297,14 @@ function windows_test_c_job($job_name, $thread_safety, $targets) { "Prepare code": stage: compile - image: registry.ddbuild.io/images/mirror/php:8.2-cli + # Same image as the package "prepare code": composer and unzip are baked in, so no apt or getcomposer.org fetch. + image: registry.ddbuild.io/images/mirror/composer:2.10.3@sha256:af98f42dfff7c68ba8d53c2164fd9fde1087b7d449514baa38c418b1f6bc4bac tags: [ "arch:amd64" ] needs: [] variables: KUBERNETES_CPU_REQUEST: 1 KUBERNETES_MEMORY_REQUEST: 2Gi before_script: - - apt update && apt install -y unzip - - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" && php composer-setup.php && mv composer.phar /usr/local/bin/composer - composer update --no-interaction script: - make generate @@ -843,6 +842,13 @@ function windows_test_c_job($job_name, $thread_safety, $targets) { KUBERNETES_MEMORY_REQUEST: 6Gi KUBERNETES_MEMORY_LIMIT: 6Gi + + script: + # Build FrankenPHP as its own step, in the proxy-free env the harness gave it: from inside PHPUnit its exit 75 is lost. + - unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy + - tooling/bin/install-frankenphp.sh || exit $? + - !reference [.cli_integration_test, script] + /etc/apt/sources.list.d/php.list' - retry_or_tempfail apt-get update - retry_or_tempfail apt-get install -y \ + retry_or_tempfail apt_get update + retry_or_tempfail apt_get install -y \ php${PHP_VERSION}-cli \ php${PHP_VERSION}-curl \ php${PHP_VERSION}-fpm \ @@ -50,7 +53,7 @@ elif [ "${INSTALL_MODE}" = "sury" ]; then WWW_CONF=/etc/php/${PHP_VERSION}/fpm/pool.d/www.conf PHP_FPM_BIN=php-fpm${PHP_VERSION} if [ "${PHP_VERSION}" != "8.5" ]; then - retry_or_tempfail apt-get install -y \ + retry_or_tempfail apt_get install -y \ php${PHP_VERSION}-opcache fi else diff --git a/dockerfiles/verify_packages/tar_gz/install.sh b/dockerfiles/verify_packages/tar_gz/install.sh index dbfdf48aa0b..525b888847f 100755 --- a/dockerfiles/verify_packages/tar_gz/install.sh +++ b/dockerfiles/verify_packages/tar_gz/install.sh @@ -19,9 +19,12 @@ retry_or_tempfail() { done } +# Bound apt: a stalled mirror fetch otherwise hangs until the 1 h job timeout. +apt_get() { timeout 300 apt-get -o Acquire::http::Timeout=30 -o Acquire::Retries=3 "$@"; } + # Common installations -retry_or_tempfail apt-get update -retry_or_tempfail apt-get install -y \ +retry_or_tempfail apt_get update +retry_or_tempfail apt_get install -y \ apt-transport-https \ lsb-release \ ca-certificates \ @@ -33,8 +36,8 @@ retry_or_tempfail apt-get install -y \ retry_or_tempfail curl -sSfL -o /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg sh -c 'echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list' -retry_or_tempfail apt-get update -retry_or_tempfail apt-get install -y \ +retry_or_tempfail apt_get update +retry_or_tempfail apt_get install -y \ php${PHP_VERSION}-cli \ php${PHP_VERSION}-curl \ php${PHP_VERSION}-fpm \ diff --git a/dockerfiles/verify_packages/verify_windows.ps1 b/dockerfiles/verify_packages/verify_windows.ps1 index d09276413eb..a333652df54 100644 --- a/dockerfiles/verify_packages/verify_windows.ps1 +++ b/dockerfiles/verify_packages/verify_windows.ps1 @@ -13,6 +13,8 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } choco install -y 7zip if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } refreshenv +# choco exits 0 after a feed error with "installed 0/0 packages"; treat a missing php as infra (75 is retried). +if (-not (Get-Command php -ErrorAction SilentlyContinue)) { Write-Host "php is not installed after choco install (feed failure?); exiting 75 so GitLab auto-retries"; exit 75 } php build/packages/datadog-setup.php --php-bin=all --file=$(ls build/packages/dd-library-php-*-x86_64-windows.tar.gz) if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/tests/Sapi/OctaneServer/OctaneServer.php b/tests/Sapi/OctaneServer/OctaneServer.php index 55ed8da2215..693148681e9 100644 --- a/tests/Sapi/OctaneServer/OctaneServer.php +++ b/tests/Sapi/OctaneServer/OctaneServer.php @@ -71,8 +71,10 @@ public function start() $this->envs["DD_TRACE_AGENT_TEST_SESSION_TOKEN"] = $token; } + // Pin the pool like the Swoole and RoadRunner harnesses: Octane's "auto" boots 2 x node vCPUs + // Laravel workers (256 on a 128-vCPU CI node), which OOMs the job. $cmd = sprintf( - PHP_BINARY . ' %s %s octane:start --server=swoole --host=%s --port=%d', + PHP_BINARY . ' %s %s octane:start --server=swoole --host=%s --port=%d --workers=2 --task-workers=1', new IniSerializer($this->inis), $this->artisanFile, $this->host, diff --git a/tooling/bin/install-frankenphp.sh b/tooling/bin/install-frankenphp.sh index 2588e6bb0f1..2c4742aa6aa 100755 --- a/tooling/bin/install-frankenphp.sh +++ b/tooling/bin/install-frankenphp.sh @@ -14,7 +14,8 @@ FRANKENPHP_SRC_DIR=/usr/local/src/frankenphp rm -rf $FRANKENPHP_SRC_DIR mkdir -p $FRANKENPHP_SRC_DIR -curl -fLo /tmp/frankenphp.tar.gz ${frankenphpTarGzUrl} +# Retries transient failures (Depot 5xx) and then exits 75, which GitLab retries. +"$(dirname "${BASH_SOURCE[0]}")/../../.gitlab/download-file.sh" "${frankenphpTarGzUrl}" /tmp/frankenphp.tar.gz tar xf /tmp/frankenphp.tar.gz -C ${FRANKENPHP_SRC_DIR} --strip-components=1 rm -f /tmp/frankenphp.tar.gz