diff --git a/tools/ci-status-org-wide.sh b/tools/ci-status-org-wide.sh new file mode 100755 index 000000000..c1057aa4b --- /dev/null +++ b/tools/ci-status-org-wide.sh @@ -0,0 +1,493 @@ +#!/bin/bash +# +# Report CI health of every repository in a GitHub organisation, on its +# default branch. +# +# The checks are deliberately strict: a repository is only reported green when +# we could actually verify that it is green. Anything unverifiable -- an API +# error, a truncated response, an unknown check conclusion, a commit that no +# workflow ever ran on -- is reported as such instead of being rounded up to a +# pass. +# +# ORG=nodejs ./ci-status-org-wide.sh +# +# Environment: +# ORG organisation to scan (default: nodejs) +# INCLUDE_ARCHIVED also scan archived repositories (default: 0) +# INCLUDE_FORKS also scan forks (default: 0) +# CANCELLED_IS_FAIL count cancelled checks as failures (default: 0) +# INCLUDE_DYNAMIC_RUNS count Dependabot "dynamic/" runs (default: 0) +# JOBS repositories checked in parallel (default: 8) +# REPO_LIMIT max repositories to list (default: 1000) +# +# Exit status is 0 only when every scanned repository was verified green. + +set -o pipefail + +ORG="${ORG:-nodejs}" +INCLUDE_ARCHIVED="${INCLUDE_ARCHIVED:-0}" +INCLUDE_FORKS="${INCLUDE_FORKS:-0}" +CANCELLED_IS_FAIL="${CANCELLED_IS_FAIL:-0}" +INCLUDE_DYNAMIC_RUNS="${INCLUDE_DYNAMIC_RUNS:-0}" +JOBS="${JOBS:-8}" +REPO_LIMIT="${REPO_LIMIT:-1000}" + +# Absolute path to self, so the xargs workers can re-enter this script +# regardless of the caller's working directory. +SELF=$(cd "$(dirname "$0")" && pwd)/$(basename "$0") + +# Conclusions that mean the check genuinely failed. "cancelled" is handled +# separately because it usually means "superseded", not "broken". +FAIL_CONCLUSIONS='"failure","timed_out","startup_failure","action_required","stale"' +PASS_CONCLUSIONS='"success","neutral","skipped"' + +# Colors (disabled when not writing to a terminal) +if [ -t 1 ]; then + BOLD=$'\033[1m'; DIM=$'\033[2m'; RESET=$'\033[0m' + GREEN=$'\033[32m'; RED=$'\033[31m'; YELLOW=$'\033[33m'; CYAN=$'\033[36m' +else + BOLD=""; DIM=""; RESET=""; GREEN=""; RED=""; YELLOW=""; CYAN="" +fi + +# --------------------------------------------------------------------------- +# API helper +# --------------------------------------------------------------------------- + +# gh_api [extra gh args...] +# +# Prints the response body and returns 0 only when the request really +# succeeded, 2 when the endpoint answered 404, 1 for anything else. gh writes the error body to stdout on a 4xx/5xx, so a plain +# `gh api ... 2>/dev/null` leaves jq looking at an error object and reading it +# as "zero checks, therefore fine" -- which is how unverifiable repositories +# used to be reported green. Transient failures are retried. +gh_api() { + local path="$1"; shift + local attempt out rc + + for attempt in 1 2 3; do + out=$(gh api "$path" "$@" 2>/dev/null) + rc=$? + + if [ $rc -eq 0 ] && [ -n "$out" ] && jq -e . >/dev/null 2>&1 <<< "$out"; then + # An error object gh happened to exit 0 on is still an error. + if jq -e '(if type == "array" then (.[0] // {}) else . end) + | type == "object" and has("message") and has("documentation_url")' \ + >/dev/null 2>&1 <<< "$out"; then + printf '%s' "$out" + api_is_404 "$out" && return 2 + return 1 + fi + printf '%s' "$out" + return 0 + fi + + # 4xx will not fix itself; only back off for empty or 5xx responses. + case "$out" in + *'"status": "4'*|*'"status":"4'*) break ;; + esac + sleep $(( attempt * 2 )) + done + + printf '%s' "$out" + api_is_404 "$out" && return 2 + return 1 +} + +# A 404 on an /actions/ endpoint means Actions is switched off for the +# repository, not that the repository is unverifiable. +api_is_404() { + case "$1" in + *'"status": "404"'*|*'"status":"404"'*|*'Not Found'*) return 0 ;; + esac + return 1 +} + +# Condense an error body into one line of explanation. +api_error() { + local msg + msg=$(jq -r '(if type == "array" then (.[0] // {}) else . end) | .message? // empty' \ + 2>/dev/null <<< "$1" | head -1) + printf '%s' "${msg:-unreachable}" +} + +# --------------------------------------------------------------------------- +# Per-repository check. Emits a single "\t