New tasks for cleaning up GCS and S3 buckets

tasks/empty-gcs-bucket/task

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${GCP_SERVICE_ACCOUNT_JSON:?}"
+: "${VARS_FILE:?}"
+: "${BUCKET_KEY:?}"
+
+bucket="$(yq -r ".${BUCKET_KEY}" "vars-file/${VARS_FILE}")"
+if [ -z "${bucket}" ] || [ "${bucket}" == "null" ]; then
+  echo "Could not read key '${BUCKET_KEY}' from vars-file/${VARS_FILE}" >&2
+  exit 1
+fi
+
+key_file="$(mktemp)"
+trap 'rm -f "${key_file}"' EXIT
+
+set +x
+echo "${GCP_SERVICE_ACCOUNT_JSON}" > "${key_file}"
+set -x
+
+service_account_email="$(jq -r .client_email "${key_file}")"
+gcloud auth activate-service-account "${service_account_email}" --key-file="${key_file}"
+
+object_glob="gs://${bucket}/**"
+
+# Empty the bucket's objects, but never delete the bucket resource itself.
+if gcloud storage ls "${object_glob}" >/dev/null 2>&1; then
+  gcloud storage rm "${object_glob}"
+else
+  echo "No objects found in gs://${bucket}; nothing to delete"
+fi

tasks/empty-gcs-bucket/task.yml

@@ -0,0 +1,19 @@
+---
+platform: linux
+
+image_resource:
+  type: registry-image
+  source:
+    repository: cloudfoundry/bbl-deployment
+
+inputs:
+- name: runtime-ci
+- name: vars-file
+
+params:
+  GCP_SERVICE_ACCOUNT_JSON:
+  VARS_FILE:
+  BUCKET_KEY:
+
+run:
+  path: runtime-ci/tasks/empty-gcs-bucket/task

tasks/empty-s3-bucket/task

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${VARS_FILE:?}"
+: "${BUCKET_KEY:?}"
+
+vars_path="vars-file/${VARS_FILE}"
+
+read_key() {
+  local key="$1"
+  local value
+  value="$(yq -r ".${key}" "${vars_path}")"
+  if [ -z "${value}" ] || [ "${value}" == "null" ]; then
+    echo "Could not read key '${key}' from ${vars_path}" >&2
+    exit 1
+  fi
+  echo "${value}"
+}
+
+bucket="$(read_key "${BUCKET_KEY}")"
+
+set +x
+AWS_ACCESS_KEY_ID="$(read_key aws_access_key_id)"
+AWS_SECRET_ACCESS_KEY="$(read_key aws_secret_access_key)"
+export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+set -x
+
+AWS_REGION="$(read_key region)"
+export AWS_REGION
+
+# Empty the bucket's objects, but never delete the bucket resource itself.
+# `aws s3 rm --recursive` is a no-op (exit 0) when the bucket is already empty.
+aws s3 rm "s3://${bucket}" --recursive

tasks/empty-s3-bucket/task.yml

@@ -0,0 +1,18 @@
+---
+platform: linux
+
+image_resource:
+  type: registry-image
+  source:
+    repository: cloudfoundry/bbl-deployment
+
+inputs:
+- name: runtime-ci
+- name: vars-file
+
+params:
+  VARS_FILE:
+  BUCKET_KEY:
+
+run:
+  path: runtime-ci/tasks/empty-s3-bucket/task