From 5cf54e0e41d0ab93dd3db99272cf365a0686c31f Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Thu, 15 Oct 2026 10:08:08 +0200 Subject: [PATCH] New tasks for cleaning up GCS and S3 buckets --- tasks/empty-gcs-bucket/task | 32 +++++++++++++++++++++++++++++++ tasks/empty-gcs-bucket/task.yml | 19 ++++++++++++++++++ tasks/empty-s3-bucket/task | 34 +++++++++++++++++++++++++++++++++ tasks/empty-s3-bucket/task.yml | 18 +++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100755 tasks/empty-gcs-bucket/task create mode 100644 tasks/empty-gcs-bucket/task.yml create mode 100755 tasks/empty-s3-bucket/task create mode 100644 tasks/empty-s3-bucket/task.yml diff --git a/tasks/empty-gcs-bucket/task b/tasks/empty-gcs-bucket/task new file mode 100755 index 00000000..f7b02865 --- /dev/null +++ b/tasks/empty-gcs-bucket/task @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${GCP_SERVICE_ACCOUNT_JSON:?}" +: "${VARS_FILE:?}" +: "${BUCKET_KEY:?}" + +bucket="$(yq -r ".${BUCKET_KEY}" "vars-file/${VARS_FILE}")" +if [ -z "${bucket}" ] || [ "${bucket}" == "null" ]; then + echo "Could not read key '${BUCKET_KEY}' from vars-file/${VARS_FILE}" >&2 + exit 1 +fi + +key_file="$(mktemp)" +trap 'rm -f "${key_file}"' EXIT + +set +x +echo "${GCP_SERVICE_ACCOUNT_JSON}" > "${key_file}" +set -x + +service_account_email="$(jq -r .client_email "${key_file}")" +gcloud auth activate-service-account "${service_account_email}" --key-file="${key_file}" + +object_glob="gs://${bucket}/**" + +# Empty the bucket's objects, but never delete the bucket resource itself. +if gcloud storage ls "${object_glob}" >/dev/null 2>&1; then + gcloud storage rm "${object_glob}" +else + echo "No objects found in gs://${bucket}; nothing to delete" +fi diff --git a/tasks/empty-gcs-bucket/task.yml b/tasks/empty-gcs-bucket/task.yml new file mode 100644 index 00000000..7cb4e358 --- /dev/null +++ b/tasks/empty-gcs-bucket/task.yml @@ -0,0 +1,19 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: cloudfoundry/bbl-deployment + +inputs: +- name: runtime-ci +- name: vars-file + +params: + GCP_SERVICE_ACCOUNT_JSON: + VARS_FILE: + BUCKET_KEY: + +run: + path: runtime-ci/tasks/empty-gcs-bucket/task diff --git a/tasks/empty-s3-bucket/task b/tasks/empty-s3-bucket/task new file mode 100755 index 00000000..f176e0ee --- /dev/null +++ b/tasks/empty-s3-bucket/task @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${VARS_FILE:?}" +: "${BUCKET_KEY:?}" + +vars_path="vars-file/${VARS_FILE}" + +read_key() { + local key="$1" + local value + value="$(yq -r ".${key}" "${vars_path}")" + if [ -z "${value}" ] || [ "${value}" == "null" ]; then + echo "Could not read key '${key}' from ${vars_path}" >&2 + exit 1 + fi + echo "${value}" +} + +bucket="$(read_key "${BUCKET_KEY}")" + +set +x +AWS_ACCESS_KEY_ID="$(read_key aws_access_key_id)" +AWS_SECRET_ACCESS_KEY="$(read_key aws_secret_access_key)" +export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +set -x + +AWS_REGION="$(read_key region)" +export AWS_REGION + +# Empty the bucket's objects, but never delete the bucket resource itself. +# `aws s3 rm --recursive` is a no-op (exit 0) when the bucket is already empty. +aws s3 rm "s3://${bucket}" --recursive diff --git a/tasks/empty-s3-bucket/task.yml b/tasks/empty-s3-bucket/task.yml new file mode 100644 index 00000000..d15d1bd4 --- /dev/null +++ b/tasks/empty-s3-bucket/task.yml @@ -0,0 +1,18 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: cloudfoundry/bbl-deployment + +inputs: +- name: runtime-ci +- name: vars-file + +params: + VARS_FILE: + BUCKET_KEY: + +run: + path: runtime-ci/tasks/empty-s3-bucket/task