lib.Makefile

# Find path to the repo root dir (i.e. this files's dir).  Must be first in the file, before including anything.
# Use Make builtins instead of `dirname` so this works under PowerShell on Windows agents too.
REPO_DIR := $(patsubst %/,%,$(dir $(realpath $(lastword $(MAKEFILE_LIST)))))

# Always install the git hooks to prevent publishing closed source code to a non-private repo.
# The install script is bash-only; skip on Windows agents where every $(shell ...) call would
# spawn a PowerShell that adds startup latency to every sub-make.
ifneq ($(OS),Windows_NT)
install_hooks := $(shell $(REPO_DIR)/hack/install-git-hooks)
endif

# Disable built-in rules
.SUFFIXES:

# Shortcut targets
default: build

## Build binary for current platform
all: build

## Run the tests for the current platform/architecture
test: ut fv st

###############################################################################
# Both native and cross architecture builds are supported.
# The target architecture is select by setting the ARCH variable.
# When ARCH is undefined it is set to the detected host architecture.
# When ARCH differs from the host architecture a crossbuild will be performed.
ARCHES ?= amd64 arm64

# list of arches *not* to build when doing *-all
EXCLUDEARCH?=ppc64le s390x
VALIDARCHES = $(filter-out $(EXCLUDEARCH),$(ARCHES))

# BUILDARCH is the host architecture
# ARCH is the target architecture
# we need to keep track of them separately
# Note: OS is always set on Windows
ifeq ($(OS),Windows_NT)
BUILDARCH = x86_64
BUILDOS = Windows
else
BUILDARCH ?= $(shell uname -m)
BUILDOS ?= $(shell uname -s | tr A-Z a-z)
endif

# canonicalized names for host architecture
ifeq ($(BUILDARCH),aarch64)
	BUILDARCH=arm64
endif
ifeq ($(BUILDARCH),x86_64)
	BUILDARCH=amd64
endif

# unless otherwise set, I am building for my own architecture, i.e. not cross-compiling
ARCH ?= $(BUILDARCH)

# canonicalized names for target architecture
ifeq ($(ARCH),aarch64)
	override ARCH=arm64
endif
ifeq ($(ARCH),x86_64)
	override ARCH=amd64
endif

# The list of sub-projects which build non-cluster host RPMs
NON_CLUSTER_HOST_SUBDIRS := selinux fluent-bit node

# detect the local outbound ip address (only used by FV/etcd targets that don't run on Windows)
ifneq ($(OS),Windows_NT)
LOCAL_IP_ENV?=$(shell ip route get 8.8.8.8 | head -1 | awk '{print $$7}')
endif

LATEST_IMAGE_TAG?=latest

# these macros create a list of valid architectures for pushing manifests
comma := ,
ifneq ($(OS),Windows_NT)
double_quote := $(shell echo '"')
endif

## Targets used when cross building.
.PHONY: native register
native:
ifneq ($(BUILDARCH),$(ARCH))
	@echo "Target $(MAKECMDGOALS)" is not supported when cross building! && false
endif

# Enable binfmt adding support for miscellaneous binary formats.
# This is only needed when running non-native binaries.
# Marked .PHONY since it produces no file; use as an order-only prerequisite
# (| register) to avoid triggering unnecessary rebuilds.
.PHONY: register
register:
ifneq ($(BUILDARCH),$(ARCH))
	docker run --privileged --rm calico/binfmt:qemu-v10.2.2 --install all || true
endif

# If this is a release, also tag and push additional images.
ifeq ($(RELEASE),true)
PUSH_IMAGES+=$(RELEASE_IMAGES)
endif

DOCKERHUB_REGISTRY ?=registry.hub.docker.com
# filter-registry filters out registries we don't want to include when tagging / pushing docker images. For instance,
# we don't include the registry name when pushing to docker hub because that registry is the default.
filter-registry ?= $(if $(filter-out $(1),$(DOCKERHUB_REGISTRY)),$(1)/)

# Convenience function to get the first dev image repo in the list.
DEV_REGISTRY ?= $(firstword $(DEV_REGISTRIES))

# remove from the list to push to manifest any registries that do not support multi-arch
MANIFEST_REGISTRIES         ?= $(DEV_REGISTRIES)

# The registry when running CI/CD pipelines.
THIRD_PARTY_REGISTRY_CI=gcr.io/unique-caldron-775/third-party-ci
THIRD_PARTY_REGISTRY_CD=gcr.io/unique-caldron-775/cnx/tigera/third-party
# THIRD_PARTY_REGISTRY configures the third-party registry that serves intermediate base image
# for some Calico Enterprise components. They are never released directly to public.
# Only master and release-calient-* branches have third-party base images published under
# their branch name. For any other branch (feature branches, or PRs whose base is a feature
# branch such as stacked PRs), fall back to the master-tagged images.
THIRD_PARTY_RELEASE_BRANCH ?= $(if $(filter master release-calient-%,$(SEMAPHORE_GIT_BRANCH)),$(SEMAPHORE_GIT_BRANCH),master)

# cd-common publishes every image at BRANCH_NAME, so that is the tag a build which names
# no version of its own can pull. CI exports it; fall back to the checked-out branch, or
# for a PR to its base, which is what SEMAPHORE_GIT_BRANCH holds.
BRANCH_IMAGE_TAG ?= $(if $(BRANCH_NAME),$(BRANCH_NAME),$(if $(SEMAPHORE_GIT_BRANCH),$(SEMAPHORE_GIT_BRANCH),$(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)))
ifeq ($(SEMAPHORE_GIT_REF_TYPE), branch)
    # on master and release-calient branches
    THIRD_PARTY_REGISTRY?=$(THIRD_PARTY_REGISTRY_CD)
else ifeq ($(SEMAPHORE_GIT_REF_TYPE), pull-request)
    # on pull requests
    THIRD_PARTY_REGISTRY?=$(THIRD_PARTY_REGISTRY_CI)
else
    # Local builds. The Semaphore git-context vars aren't set, so default to the
    # CD registry, which serves the master-tagged base images that local enterprise
    # builds (fluentd, elasticsearch, deep-packet-inspection, kibana) pull from.
    THIRD_PARTY_REGISTRY?=$(THIRD_PARTY_REGISTRY_CD)
endif

THIRD_PARTY_BASE_IMAGES_TO_RETAG = elasticsearch kibana snort3
release-retag-third-party-base-images: var-require-one-of-CONFIRM-DRYRUN var-require-all-RELEASE_BRANCH
	$(MAKE) $(addprefix release-retag-third-party-base-images-,$(THIRD_PARTY_BASE_IMAGES_TO_RETAG))

release-retag-third-party-base-images-%: var-require-one-of-CONFIRM-DRYRUN var-require-all-RELEASE_BRANCH bin/crane
	$(MAKE) -C third_party/$* release-retag-image

PUSH_MANIFEST_IMAGES := $(foreach registry,$(MANIFEST_REGISTRIES),$(foreach image,$(BUILD_IMAGES),$(call filter-registry,$(registry))$(image)))

# location of docker credentials to push manifests
DOCKER_CONFIG ?= $(HOME)/.docker/config.json

# If a repository still relies on vendoring, it must set GOMOD_VENDOR to "true".
# If that's not the case and we're running in CI, set -mod=readonly to prevent builds
# from being flagged as dirty due to updates in go.mod or go.sum _except_ for:
# - for local builds, which _require_ a change to go.mod.
# - the targets 'commit-pin-updates' and  'golangci-lint' which require
#   updating go.mod and/or go.sum
SKIP_GOMOD_READONLY_FLAG =
ifeq ($(MAKECMDGOALS),commit-pin-updates)
	SKIP_GOMOD_READONLY_FLAG = yes
endif
ifeq ($(MAKECMDGOALS),golangci-lint)
	SKIP_GOMOD_READONLY_FLAG = yes
endif

ifeq ($(GOMOD_VENDOR),true)
	GOFLAGS?="-mod=vendor"
else
ifeq ($(CI),true)
ifndef SKIP_GOMOD_READONLY_FLAG
	GOFLAGS?="-mod=readonly"
endif
endif
endif

# Optional cap on go build/test package parallelism. Appended to GOFLAGS so
# it flows through every docker invocation that already passes GOFLAGS in.
# Useful for limiting memory pressure when running multiple parallel builds
# on a workstation (each in-flight package can fork its own compiler).
ifneq ($(GO_BUILD_PARALLELISM),)
GOFLAGS := $(GOFLAGS) -p=$(GO_BUILD_PARALLELISM)
endif

# Outer parallelism for image / kind-build-images / kind-reload. Each parallel
# job spawns a docker go-build container, so `-j$(nproc)` on a workstation with
# limited RAM (e.g. 32G running alongside an IDE/LSP/AI session) will thrash
# into swap. Default to a conservative 4. Raise via NUM_BUILD_JOBS=N for a
# bigger machine.
NUM_BUILD_JOBS ?= 4

# For building, we use the go-build image for the *host* architecture, even if the target is different
# the one for the host should contain all the necessary cross-compilation tools
# we do not need to use the arch since go-build:v0.15 now is multi-arch manifest
GO_BUILD_IMAGE ?= calico/go-build
CALICO_BUILD    = $(GO_BUILD_IMAGE):$(GO_BUILD_VER)

RUST_BUILD_IMAGE ?= calico/rust-build
CALICO_RUST_BUILD = $(RUST_BUILD_IMAGE):$(RUST_BUILD_VER)

# Cross-compilation support using clang.  When building on amd64 for a different
# target architecture, we use clang as a cross-compiler (it natively supports all
# targets) with a per-arch sysroot containing the target libraries.  This avoids
# slow QEMU emulation for CGO builds.
#
# Map Go ARCH names to clang target triples.
CLANG_CROSS_TRIPLE_arm64   := aarch64-linux-gnu
CLANG_CROSS_TRIPLE_ppc64le := powerpc64le-linux-gnu
CLANG_CROSS_TRIPLE_s390x   := s390x-linux-gnu

# Rust target triple (long form). Injected as CARGO_BUILD_TARGET so cargo
# cross-compiles transparently. Linker/sysroot side uses CROSS_TRIPLE below.
RUST_TARGET_amd64   := x86_64-unknown-linux-gnu
RUST_TARGET_arm64   := aarch64-unknown-linux-gnu
RUST_TARGET         := $(RUST_TARGET_$(ARCH))

# Set CROSS_CC and CROSS_SYSROOT when cross-compiling from amd64.
ifeq ($(BUILDARCH),amd64)
ifneq ($(ARCH),amd64)
ifneq ($(CLANG_CROSS_TRIPLE_$(ARCH)),)
CROSS_TRIPLE := $(CLANG_CROSS_TRIPLE_$(ARCH))
CROSS_SYSROOT := /usr/$(CROSS_TRIPLE)/sys-root
CROSS_CC := clang --target=$(CROSS_TRIPLE) --sysroot=$(CROSS_SYSROOT)
CROSS_LDFLAGS := -fuse-ld=lld
endif
endif
endif

# The build macros below use $(DOCKER_GO_BUILD_RECURSIVE) (defined with DOCKER_RUN, further down)
# as their container prefix; they keep their env and mkdir inside the sh -c so it is the
# only container-vs-native difference.

# Use this when building binaries that need cgo (e.g. for libbpf).
define build_cgo_binary
	$(DOCKER_GO_BUILD_RECURSIVE) \
		sh -c '$(GIT_CONFIG_SSH) mkdir -p $(dir $(2)) && CGO_ENABLED=1 $(if $(CROSS_CC),CC="$(CROSS_CC)") CGO_CFLAGS=$(CGO_CFLAGS) CGO_LDFLAGS=$(CGO_LDFLAGS) go build -o $(2) $(if $(BUILD_TAGS),-tags $(BUILD_TAGS)) -v -buildvcs=false -ldflags "$(LDFLAGS) -s -w" $(1)'
endef

# For binaries that do not require cgo.
define build_binary
	$(DOCKER_GO_BUILD_RECURSIVE) \
		sh -c '$(GIT_CONFIG_SSH) mkdir -p $(dir $(2)) && CGO_ENABLED=0 go build -o $(2) $(if $(BUILD_TAGS),-tags $(BUILD_TAGS)) -v -buildvcs=false -ldflags "$(LDFLAGS) -s -w" $(1)'
endef

define build_binary_dir
	$(DOCKER_GO_BUILD_RECURSIVE) \
		sh -c '$(GIT_CONFIG_SSH) mkdir -p $(1)/$(dir $(3)) && CGO_ENABLED=0 go build -C $(1) -o $(3) $(if $(BUILD_TAGS),-tags $(BUILD_TAGS)) -v -buildvcs=false -ldflags "$(LDFLAGS) -s -w" $(2)'
endef

# GOEXPERIMENT=nodwarf5 is required to disable DWARF5 debug format which is incompatible
# with current version of MinGW toolchain. Without this, binaries will fail with
# "not a valid application for this OS platform" error.
# https://github.com/golang/go/issues/75077

# Docker env for cgo Windows (amd64) cross-compilation via the MinGW toolchain.
# Shared by build_cgo_windows_binary (go build) and by cgo test-binary targets
# that need the same toolchain but run `go test -c` (e.g. felix's fv/win-fv.exe).
CGO_WINDOWS_DOCKER_ENV = \
	-e CC=x86_64-w64-mingw32-gcc \
	-e CGO_ENABLED=1 \
	-e GOARCH=amd64 \
	-e GOOS=windows \
	-e GOEXPERIMENT=nodwarf5

# For windows builds that require cgo.
define build_cgo_windows_binary
	$(DOCKER_RUN) $(CGO_WINDOWS_DOCKER_ENV) $(CALICO_BUILD) \
		sh -c '$(GIT_CONFIG_SSH) go build -o $(2) $(if $(BUILD_TAGS),-tags $(BUILD_TAGS)) -v -buildvcs=false -ldflags "$(LDFLAGS) -s -w" $(1)'
endef

# For windows builds that do not require cgo.
define build_windows_binary
	$(DOCKER_RUN) \
		-e CGO_ENABLED=0 \
		-e GOARCH=amd64 \
		-e GOOS=windows \
		-e GOEXPERIMENT=nodwarf5 \
		$(CALICO_BUILD) \
		sh -c '$(GIT_CONFIG_SSH) go build -o $(2) $(if $(BUILD_TAGS),-tags $(BUILD_TAGS)) -v -buildvcs=false -ldflags "$(LDFLAGS) -s -w" $(1)'
endef

# Images used in build / test across multiple directories.
ETCD_IMAGE ?= quay.io/coreos/etcd:$(ETCD_VERSION)-$(ARCH)
ifeq ($(BUILDARCH),amd64)
	# *-amd64 tagged images for etcd are not available until v3.5.0
	ETCD_IMAGE = quay.io/coreos/etcd:$(ETCD_VERSION)
endif

UBI_IMAGE ?= registry.access.redhat.com/ubi10/ubi-minimal:latest

ifeq ($(GIT_USE_SSH),true)
	GIT_CONFIG_SSH ?= git config --global url."ssh://git@github.com/".insteadOf "https://github.com/";
endif

# Get version from git. We allow setting this manually for the hashrelease process.
# By default, includes commit count and hash (--long). During releases (RELEASE=true),
# only the tag is used without the commit count suffix.
#
# Skip on Windows: these LDFLAGS-related vars use bash `||` fallbacks that PowerShell
# can't parse, and pure-Docker Windows image builds don't link Go binaries that
# embed buildinfo. Each $(shell git ...) call would otherwise spawn PowerShell on
# every sub-make recursion, multiplying parse time noticeably.
ifneq ($(OS),Windows_NT)
GIT_VERSION ?= $(shell git describe --tags --dirty --always --abbrev=12 --long)
ifeq ($(RELEASE),true)
	GIT_VERSION := $(shell git describe --tags --dirty --always --abbrev=12)
endif

# Figure out version information.  To support builds from release tarballs, we default to
# <unknown> if this isn't a git checkout.
GIT_COMMIT:=$(shell git rev-parse HEAD || echo '<unknown>')
BUILD_ID:=$(shell git rev-parse HEAD || uuidgen | sed 's/-//g')

# Lazily set the git version we embed into the binaries we build. We want the
# git tag at the time we build the binary.
# Variables elsewhere that depend on this (such as LDFLAGS) must also be lazy.
GIT_DESCRIPTION=$(shell git describe --tags --dirty --always --abbrev=12 || echo '<unknown>')
ENTERPRISE_VERSION?=$(call git-release-tag-from-dev-tag)
endif

# Calculate a timestamp for any build artifacts.
ifneq ($(OS),Windows_NT)
DATE:=$(shell date -u +'%FT%T%z')
endif

# Common linker flags.
#
# We use -X to insert the version information into the placeholder variables
# in the buildinfo package.
LDFLAGS=-X github.com/projectcalico/calico/pkg/buildinfo.Version=$(GIT_DESCRIPTION) \
	-X github.com/projectcalico/calico/pkg/buildinfo.BuildDate=$(DATE) \
	-X github.com/projectcalico/calico/pkg/buildinfo.GitRevision=$(GIT_COMMIT)

# Add in flags that are not used in OSS builds.
LDFLAGS+=-X github.com/projectcalico/calico/pkg/buildinfo.EnterpriseReleaseVersion=$(ENTERPRISE_VERSION)
LDFLAGS+=-X github.com/projectcalico/calico/pkg/buildinfo.OpenSourceBaseVersion=$(CALICO_VERSION)

# We use -B to insert a build ID note into the executable, without which, the
# RPM build tools complain.
ifneq ($(BUILDOS),darwin)
	LDFLAGS+=-B 0x$(BUILD_ID)
endif

# Figure out the users UID/GID.  These are needed to run docker containers
# as the current user and ensure that files built inside containers are
# owned by the current user.
ifneq ($(OS),Windows_NT)
LOCAL_USER_ID:=$(shell id -u)
LOCAL_GROUP_ID:=$(shell id -g)
endif

ifeq ("$(LOCAL_USER_ID)", "0")
# The build needs to run as root.
EXTRA_DOCKER_ARGS+=-e RUN_AS_ROOT='true'
endif

# Allow the ssh auth sock to be mapped into the build container.
ifdef SSH_AUTH_SOCK
	EXTRA_DOCKER_ARGS += -v $(SSH_AUTH_SOCK):/ssh-agent --env SSH_AUTH_SOCK=/ssh-agent
endif

# Volume-mount gopath into the build container to cache go module's packages. If the environment is using multiple
# comma-separated directories for gopath, use the first one, as that is the default one used by go modules.
ifneq ($(GOPATH),)
	# If the environment is using multiple comma-separated directories for gopath, use the first one, as that
	# is the default one used by go modules.
	GOMOD_CACHE = $(shell echo $(GOPATH) | cut -d':' -f1)/pkg/mod
else
	# If gopath is empty, default to $(HOME)/go.
	GOMOD_CACHE = $(HOME)/go/pkg/mod
endif

EXTRA_DOCKER_ARGS += -v $(GOMOD_CACHE):/go/pkg/mod:rw

# Optional per-build resource caps. When unset, no flags are added and the
# container has full host access (current behaviour). Useful when running
# multiple parallel builds on a workstation to avoid memory thrash.
#   DOCKER_CPUS=N           Hard cap on total CPU bandwidth (any core).
#   DOCKER_CPUSET_CPUS=0-3  Pin container to specific cores (true affinity).
#   GOMAXPROCS=N            Cap goroutine parallelism inside each go invocation
#                           (linker, vet, etc.); complements -p=N from GOFLAGS.
#   DOCKER_MEMORY=8g        Hard cap on container memory. The kernel OOM-kills
#                           the container at this limit, so a runaway compile or
#                           link fails the build ("signal: killed") rather than
#                           driving the whole host into swap thrash.
#   DOCKER_MEMORY_SWAP=8g   Cap on memory+swap combined (requires DOCKER_MEMORY).
#                           Must be >= DOCKER_MEMORY or -1; defaults to DOCKER_MEMORY
#                           (denies container swap entirely, keeps it off host swap).
ifneq ($(DOCKER_CPUS),)
EXTRA_DOCKER_ARGS += --cpus=$(DOCKER_CPUS)
endif
ifneq ($(DOCKER_CPUSET_CPUS),)
EXTRA_DOCKER_ARGS += --cpuset-cpus=$(DOCKER_CPUSET_CPUS)
endif
ifneq ($(GOMAXPROCS),)
EXTRA_DOCKER_ARGS += -e GOMAXPROCS=$(GOMAXPROCS)
endif
ifneq ($(DOCKER_MEMORY),)
EXTRA_DOCKER_ARGS += --memory=$(DOCKER_MEMORY) --memory-swap=$(if $(DOCKER_MEMORY_SWAP),$(DOCKER_MEMORY_SWAP),$(DOCKER_MEMORY))
endif

# Define go architecture flags
GOARCH_FLAGS :=-e GOARCH=$(ARCH)

ifeq ($(ARCH),amd64)
GOARCH_FLAGS += -e GOAMD64=v3
endif

# Location of certificates used in UTs.
REPO_ROOT := $(shell git rev-parse --show-toplevel)
CERTS_PATH := $(REPO_ROOT)/hack/test/certs

# Support for git worktrees.  In a worktree, .git is a file pointing to
# <main-repo>/.git/worktrees/<name>.  When Docker containers need git access,
# the main .git directory must also be mounted, and GIT_DIR / GIT_WORK_TREE
# must be set so that git can find objects and the correct working tree.
#
# Skip on Windows: this only configures Linux Docker mounts, and the bash `2>/dev/null`
# redirection PowerShell tries to interpret as `2 > /dev/null` (writing to a file at
# C:\dev\null), which fails noisily on every parse.
ifneq ($(OS),Windows_NT)
_GIT_DIR := $(shell git rev-parse --absolute-git-dir 2>/dev/null)
_GIT_COMMON_DIR := $(realpath $(shell git rev-parse --git-common-dir 2>/dev/null))
ifneq ($(_GIT_DIR),$(_GIT_COMMON_DIR))
# Running in a git worktree: mount the main .git directory at its host path
# so the worktree .git file pointer resolves inside the container.
# DOCKER_GIT_WORK_TREE can be overridden by Makefiles that mount the repo at
# a different container path (e.g. api/Makefile).
DOCKER_GIT_WORK_TREE ?= /go/src/github.com/projectcalico/calico
DOCKER_GIT_WORKTREE_ARGS := \
	-v $(_GIT_COMMON_DIR):$(_GIT_COMMON_DIR):ro \
	-e GIT_DIR=$(_GIT_DIR) \
	-e GIT_WORK_TREE=$(DOCKER_GIT_WORK_TREE)
else
DOCKER_GIT_WORKTREE_ARGS :=
endif
endif

# Configure the Calico API group to use. Projects importing this Makefile can override this variable
# if they need to.
# Supported values:
# - crd.projectcalico.org/v1 (default)
# - projectcalico.org/v3
CALICO_API_GROUP ?= crd.projectcalico.org/v1

# Where to find Calico CRD files depends on which API group we are using to back them.
CALICO_CRD_PATH ?= api/config/crd/
ifneq ($(CALICO_API_GROUP),projectcalico.org/v3)
CALICO_CRD_PATH = libcalico-go/config/crd/
endif

# The image to use for building calico/base-dependent modules (e.g. apiserver, typha).
ifdef USE_UBI_AS_CALICO_BASE
CALICO_BASE ?= $(UBI_IMAGE)
else
CALICO_BASE ?= calico/base:$(CALICO_BASE_VER)
endif

ifndef NO_DOCKER_PULL
DOCKER_PULL = --pull
else
DOCKER_PULL =
endif

# DOCKER_BUILD is the base build command used for building all images.
DOCKER_BUILD=docker buildx build --load --platform=linux/$(ARCH) $(DOCKER_PULL) \
	--build-arg CALICO_BASE=$(CALICO_BASE) \
	--build-arg GIT_VERSION=$(GIT_VERSION) \
	--build-arg UBI_IMAGE=$(UBI_IMAGE)

DOCKER_BUILD_THIRD_PARTY = $(DOCKER_BUILD) \
	--build-arg THIRD_PARTY_REGISTRY=$(THIRD_PARTY_REGISTRY) \
	--build-arg THIRD_PARTY_RELEASE_BRANCH=$(THIRD_PARTY_RELEASE_BRANCH)

fetch_file = $(REPO_ROOT)/hack/fetch-file $(1) $(2)
fetch_repo = $(REPO_ROOT)/hack/fetch-repo $(1) $(2) $(3)

# Fail a clone that needs credentials instead of blocking on git's terminal
# prompt, which hangs a CI job until the pipeline timeout.
export GIT_TERMINAL_PROMPT ?= 0

DEFAULT_GO_CACHE_PATH := $(REPO_ROOT)/.go-pkg-cache
GOENV_GOCACHE := $(strip $(shell go env GOCACHE 2>/dev/null))

# Mount source for the Go build cache, first match wins:
#   1. LOCAL_GO_PKG_CACHE from the environment
#   2. GOCACHE, when the Go tools resolve it to an absolute path (it is either
#      that or the literal "off"; a relative path would make docker -v fail)
#   3. the repo-local .go-pkg-cache
#
# override, not ?=: `LOCAL_GO_PKG_CACHE= make ...` to opt out leaves the
# variable defined-but-empty, which ?= keeps, yielding `docker run -v :/go-cache`.
ifeq ($(strip $(LOCAL_GO_PKG_CACHE)),)
override LOCAL_GO_PKG_CACHE := $(or $(filter /%,$(GOENV_GOCACHE)),$(DEFAULT_GO_CACHE_PATH))
endif

DOCKER_RUN_PRIV_NET := mkdir -p $(LOCAL_GO_PKG_CACHE) bin $(GOMOD_CACHE) && \
	docker run --rm \
		--init \
		$(EXTRA_DOCKER_ARGS) \
		$(DOCKER_GIT_WORKTREE_ARGS) \
		-e LOCAL_USER_ID=$(LOCAL_USER_ID) \
		-e GOCACHE=/go-cache \
		-e GIT_TERMINAL_PROMPT=$(GIT_TERMINAL_PROMPT) \
		$(GOARCH_FLAGS) \
		-e GOPATH=/go \
		-e OS=$(BUILDOS) \
		-e GOOS=$(BUILDOS) \
		-e CALICO_API_GROUP=$(CALICO_API_GROUP) \
		-e "GOFLAGS=$(GOFLAGS)" \
		-v $(REPO_ROOT):/go/src/github.com/projectcalico/calico:rw \
		-v $(LOCAL_GO_PKG_CACHE):/go-cache:rw \
		-w /go/src/$(PACKAGE_NAME)

DOCKER_RUN := $(DOCKER_RUN_PRIV_NET) --net=host

# Prefix for targets anchored at the repo root (the kind install's GOBIN, fix-all, ...).
# Immediate (:=) so a sub-Makefile that overrides DOCKER_RUN after this include (api/Makefile)
# can't change it -- those targets must keep the repo-root mount.
DOCKER_GO_BUILD := $(DOCKER_RUN) $(CALICO_BUILD)

# Prefix for the build_* macros. Recursive (=) so it honors a sub-Makefile's DOCKER_RUN
# override -- api/Makefile mounts itself as .../api for its own local builds (e.g. list-gnp).
DOCKER_GO_BUILD_RECURSIVE = $(DOCKER_RUN) $(CALICO_BUILD)

# The repo root as a go-build recipe sees it: the container's mount, or the checkout itself
# natively. For absolute paths handed to the build, e.g. CGO include and library dirs.
BUILD_REPO_ROOT := /go/src/github.com/projectcalico/calico

# NATIVE_GO_BUILD=true (already inside calico/go-build, e.g. a CI go-build pod): null the
# go-build prefixes so those recipes run on the host, and export the Go env the container
# used to pass via -e. Targets that spell out their own docker invocation keep
# DOCKER_RUN/CALICO_BUILD.
ifeq ($(NATIVE_GO_BUILD),true)
DOCKER_GO_BUILD :=
DOCKER_GO_BUILD_RECURSIVE :=
BUILD_REPO_ROOT := $(REPO_ROOT)
export GOARCH := $(ARCH)
export GOOS := $(BUILDOS)
export GOFLAGS := $(GOFLAGS)
ifeq ($(ARCH),amd64)
export GOAMD64 := v2
endif
endif

# Cross-compile env for Rust + cc-rs / bindgen. Same gate as the Go side.
# Key suffixes use the long Rust triple (cc-rs convention); extend for ppc64le.
#
# Build scripts reuse CFLAGS to link probe programs. -B points clang at the
# cross binutils for those links; unlike -fuse-ld, it is not reported unused on
# compile-only calls, where that warning fails -Werror feature probes such as
# those in aws-lc-sys.
ifeq ($(BUILDARCH),amd64)
ifneq ($(ARCH),amd64)
RUST_CROSS_ENV := \
	-e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=clang \
	-e CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS="-C link-arg=--target=$(CROSS_TRIPLE) -C link-arg=--sysroot=$(CROSS_SYSROOT) -C link-arg=-fuse-ld=lld" \
	-e CC_aarch64_unknown_linux_gnu=clang \
	-e CXX_aarch64_unknown_linux_gnu=clang++ \
	-e AR_aarch64_unknown_linux_gnu=$(CROSS_TRIPLE)-ar \
	-e CFLAGS_aarch64_unknown_linux_gnu="--sysroot=$(CROSS_SYSROOT) -B/usr/bin/$(CROSS_TRIPLE)-" \
	-e CXXFLAGS_aarch64_unknown_linux_gnu="--sysroot=$(CROSS_SYSROOT) -B/usr/bin/$(CROSS_TRIPLE)-" \
	-e BINDGEN_EXTRA_CLANG_ARGS_aarch64_unknown_linux_gnu="--target=$(CROSS_TRIPLE) --sysroot=$(CROSS_SYSROOT) -I$(CROSS_SYSROOT)/usr/include"
endif
endif

###############################################################################
# Calico-patched controller-gen
#
# CRD generation uses a controller-gen with Calico-specific patches (see the
# *.patch files under //hack/cmd/calico-controller-gen): NumOrString/Port/
# Protocol/DSCP int-or-string union schemas, and nullable slice-of-pointer
# elements. The projectcalico/toolchain repo bakes a (NumOrString-only) patched
# binary into the calico/go-build image; we build our own patched binary
# in-repo instead (download tarball -> apply patches -> go build), so CRD
# generation owns its patches and does not depend on the image's controller-gen.
#
# The controller-tools version is pinned inside build.sh (its VERSION line, the
# single source of truth). build.sh checks it against the controller-gen baked
# into the calico/go-build image and, on a mismatch, rewrites that line and
# fails so the bump is committed deliberately. We read the pin here cheaply (no
# container) for the cache key.
CONTROLLER_TOOLS_VERSION := $(shell sed -n 's/^VERSION="\(v[0-9][0-9.]*\)".*/\1/p' $(REPO_ROOT)/hack/cmd/calico-controller-gen/build.sh | head -1)
CONTROLLER_TOOLS_VERSION := $(or $(CONTROLLER_TOOLS_VERSION),v0.18.0)

# The binary is built into the shared Go build cache (mounted as /go-cache in
# every component container, including api/'s isolated mount). It is stamped
# with the go-build image version, the controller-tools version, and a hash of
# build.sh and the patches: bumping the image (which may carry a new
# controller-gen), the pinned version, or editing either file yields a new path
# and triggers a rebuild — and a rebuild re-runs the image-vs-pin check in
# build.sh. build.sh is hashed because checkouts sharing the cache share the
# output file, which defeats Make's prerequisite check.
CALICO_CONTROLLER_GEN_HASH := $(shell cat $(REPO_ROOT)/hack/cmd/calico-controller-gen/build.sh $(REPO_ROOT)/hack/cmd/calico-controller-gen/*.patch 2>/dev/null | sha256sum | cut -c1-12)
CALICO_CONTROLLER_GEN_STAMP := $(GO_BUILD_VER)-$(CONTROLLER_TOOLS_VERSION)-$(CALICO_CONTROLLER_GEN_HASH)
# Two views of the same file: the host path Make uses as a build target, and
# the in-container path (/go-cache is the bind-mount of LOCAL_GO_PKG_CACHE) used to
# invoke it from inside the build containers.
CALICO_CONTROLLER_GEN_BIN := $(LOCAL_GO_PKG_CACHE)/bin/calico-controller-gen-$(CALICO_CONTROLLER_GEN_STAMP)
CALICO_CONTROLLER_GEN     := /go-cache/bin/calico-controller-gen-$(CALICO_CONTROLLER_GEN_STAMP)

# Real file target (not .PHONY): Make skips it entirely — no container spin-up —
# when the binary already exists. Patch edits and version-pin bumps both land in
# the filename above (via the hash and the pinned version), so they yield a new
# target and trigger a rebuild. build.sh is an order-only prerequisite for that
# reason: its contents are already in the stamp, and comparing mtimes instead
# would rebuild whenever a fresh checkout gave build.sh a timestamp newer than a
# valid binary another checkout left in the shared cache. The recipe
# needs the repo root mounted (for build.sh and the patches), so components in
# their own module (api/) reach it via:
#   $(MAKE) -C $(REPO_ROOT) $(CALICO_CONTROLLER_GEN_BIN)
$(CALICO_CONTROLLER_GEN_BIN): | hack/cmd/calico-controller-gen/build.sh
	$(DOCKER_GO_BUILD) sh -c \
		'./hack/cmd/calico-controller-gen/build.sh $(CALICO_CONTROLLER_GEN)'
	@# `go clean -cache` only empties the cache's hex subdirectories, so stamps
	@# from superseded image/version/patch combinations would accumulate here
	@# forever. Drop the ones untouched for a month; a checkout still pinned to
	@# such a stamp just rebuilds it.
	@find $(dir $(CALICO_CONTROLLER_GEN_BIN)) -maxdepth 1 -type f \
		-name 'calico-controller-gen-*' -mtime +30 -delete 2>/dev/null || true

DOCKER_RUST_BUILD := mkdir -p bin && \
	docker run --rm \
		--init \
		--user $(LOCAL_USER_ID):$(LOCAL_GROUP_ID) \
		$(EXTRA_DOCKER_ARGS) \
		-e CARGO_BUILD_TARGET=$(RUST_TARGET) \
		$(RUST_CROSS_ENV) \
		-v $(REPO_ROOT):/rust/src/github.com/projectcalico/calico:rw \
		-w /rust/src/$(PACKAGE_NAME) \
		$(CALICO_RUST_BUILD)

# Host native build images
HOST_NATIVE_BUILD_IMAGE ?= calico/host-native-build

# Distros for which we maintain a host-native build image — discovered from
# Dockerfiles in third_party/host-native/. New distros are picked up
# automatically by adding a Dockerfile.<distro>.
HOST_NATIVE_DISTROS := $(patsubst Dockerfile.%,%,$(notdir $(wildcard $(REPO_DIR)/third_party/host-native/Dockerfile.*)))

.PHONY: host-native-build $(addprefix host-native-build-,$(HOST_NATIVE_DISTROS))

# Umbrella — build host-native images for every supported distro.
host-native-build: $(addprefix host-native-build-,$(HOST_NATIVE_DISTROS))

# Per-distro — build only the named host-native image. Package targets should
# depend on `host-native-build-<distro>` to avoid triggering bakes of images
# they don't use. Generated as explicit rules per distro because GNU make
# does not apply pattern rules to .PHONY targets.
define host_native_build_rule
host-native-build-$(1): $$(REPO_DIR)/third_party/host-native/Dockerfile.$(1)
	cd $$(REPO_DIR)/third_party/host-native && \
		docker buildx bake --load --pull \
		-f $$(REPO_DIR)/third_party/host-native/host-native-build-bake.hcl $(1)
endef
$(foreach d,$(HOST_NATIVE_DISTROS),$(eval $(call host_native_build_rule,$(d))))

DOCKER_HOST_NATIVE_RUN := docker run --rm \
	--net=host \
	--init \
	--user $(LOCAL_USER_ID):$(LOCAL_GROUP_ID) \
	$(EXTRA_DOCKER_ARGS) \
	-v $(REPO_ROOT):/go/src/github.com/projectcalico/calico:rw \
	-w /go/src/$(PACKAGE_NAME)

# This function replaces the version string in the spec template and builds the rpm package.
# Args: $(1) distro, $(2) package name, $(3) component version (passed to
# generate-package-version.sh), $(4) optional sourcedir override (relative to
# component directory; defaults to component dir itself).
# Spec templates may use @VERSION@ (the full package version) and
# @UPSTREAM_VERSION@ (the raw component version $(3), e.g. for %autosetup -n on an
# unrenamed upstream tarball). Components that don't reference these are unaffected.
define host_native_rpm_build
	$(eval version := $(shell $(REPO_ROOT)/hack/generate-package-version.sh $(REPO_ROOT) rpm "" $(3)))

	sed -e 's/@VERSION@/$(version)/g' \
	    -e 's/@UPSTREAM_VERSION@/$(3)/g' \
	    rhel/$(2).spec.in > rhel/$(2).spec

	mkdir -p package/$(1) && $(DOCKER_HOST_NATIVE_RUN) \
		$(HOST_NATIVE_BUILD_IMAGE):$(1) \
			sh -c 'rpmbuild \
				--define "_topdir $$PWD/package/$(1)" \
				--define "_sourcedir $$PWD$(if $(4),/$(4),)" \
				-ba rhel/$(2).spec'
endef

# DEB_PATCHES_DIR is the directory a component stages its quilt patches in,
# relative to the component dir. Defaults to the component dir itself for
# backwards compatibility; components that group patches in a subdirectory
# (e.g. fluent-bit) override it. The series file lists patch basenames, so the
# listing is run from inside DEB_PATCHES_DIR.
DEB_PATCHES_DIR ?= .

define host_native_deb_build
	$(eval version := $(shell $(REPO_ROOT)/hack/generate-package-version.sh $(REPO_ROOT) deb $(4) $(3)))

	$(eval timestamp := $(shell date -u '+%a, %d %b %Y %H:%M:%S +0000'))

	mkdir -p package/$(1)/$(2)/debian/patches package/$(1)/$(2)/debian/source

	sed -e 's/@VERSION@/$(version)/g' \
	    -e 's/@DISTRIBUTION@/$(4)/g' \
	    -e 's/@PACKAGE_NAME@/$(2)/g' \
	    -e 's/@TIMESTAMP@/$(timestamp)/g' \
		debian/changelog.in > package/$(1)/$(2)/debian/changelog

	cp $(DEB_PATCHES_DIR)/*.patch package/$(1)/$(2)/debian/patches/ 2>/dev/null || true
	( cd $(DEB_PATCHES_DIR) && ls *.patch 2>/dev/null ) > package/$(1)/$(2)/debian/patches/series || echo -n "" > package/$(1)/$(2)/debian/patches/series

	echo "3.0 (quilt)" > package/$(1)/$(2)/debian/source/format

	cp \
	   debian/control \
	   debian/copyright \
	   debian/rules \
	   debian/$(2).postinst \
	   $(wildcard debian/$(2).prerm) \
		package/$(1)/$(2)/debian/

	cp \
	   *.conf \
	   $(2).env \
	   $(2).service \
		package/$(1)/$(2)/debian/

	# Extra units are staged under debhelper's "<package>.<unit>.service"
	# name so dh_installsystemd decides where they land: the unit directory
	# moved from /lib to /usr/lib in trixie, and a package's units must agree.
	$(foreach u,$(wildcard $(2)-*.service),cp $(u) package/$(1)/$(2)/debian/$(2).$(u);)

	tar --strip-components=$(5) -C package/$(1)/$(2) -xf $(2).tar.gz

	mkdir -p package/$(1) && $(DOCKER_HOST_NATIVE_RUN) \
		$(HOST_NATIVE_BUILD_IMAGE):$(1) \
			sh -c 'cd package/$(1)/$(2) && debuild -us -uc -b'
endef

###############################################################################
# Source file dependency tracking via deps.txt
#
# Each component's deps.txt contains both external module dependencies and
# local in-repo package directories (prefixed with "local:"). If deps.txt
# has local entries, SRC_FILES is automatically populated with all .go files
# in those directories. Components can append extra non-Go dependencies
# (e.g., BPF .c/.h files) after including lib.Makefile.
#
# IMAGE_DEPS lists non-Go files that the Docker image depends on (Dockerfiles,
# config templates, scripts, etc.). Components should override or append to
# this variable and include $(IMAGE_DEPS) in their .image.created prereqs.
#
# Skip on Windows: this is dependency tracking for Linux Go-based image stamps;
# Windows components that build pure Docker images (no Go binary) don't need it.
# Each $(shell find ... grep ... cut ...) call would otherwise spawn PowerShell
# (no find/grep/cut available) on every sub-make recursion. With ~20 components
# referenced from .image.created-* prerequisites below, this was the dominant
# contributor to the Windows publish job hitting Semaphore's 3h timeout.
###############################################################################
IMAGE_DEPS ?= Dockerfile

ifneq ($(OS),Windows_NT)
ifneq ($(wildcard deps.txt),)
SRC_FILES := $(shell find $(addprefix $(REPO_ROOT)/,$(shell grep '^local:' deps.txt | cut -d: -f2-)) -name '*.go' 2>/dev/null)
endif

# Expand a component's deps.txt local entries to the list of .go files.
# Usage: $(call local-deps-go-files,<component-dir>)
local-deps-go-files = $(shell find $(addprefix $(REPO_ROOT)/,$(shell grep '^local:' $(REPO_ROOT)/$(1)/deps.txt | cut -d: -f2-)) -name '*.go' 2>/dev/null)
endif

# A target that does nothing but it always stale, used to force a rebuild on certain targets based on some non-file criteria.
.PHONY: force-rebuild
force-rebuild:

###############################################################################
# Updating pins
#   the repo importing this Makefile _must_ define the update-pins target
#   for example:
#     update-pins: update-libcalico-pin update-typha-pin
###############################################################################
PIN_BRANCH?=$(shell git rev-parse --abbrev-ref HEAD)

# The docker entrypoint script might echo output that could be included in the output of the following command, so this
# prefixes the commit tag with "commit-tag:" so can reliable get the commit tag from the output.
define get_remote_version
	$(shell $(DOCKER_RUN) $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) echo "commit-tag:$$(git ls-remote https://$(1) $(2) | cut -f1)"' | awk -F "commit-tag:" '{print $$2}')
endef

# update_pin updates the given package's version to the latest available in the specified repo and branch.
# $(1) should be the name of the package, $(2) and $(3) the repository and branch from which to update it.
# If $(4) is specified it's treated as the module version and use in the go get -d command.
define update_pin
	$(eval new_ver := $(call get_remote_version,$(2),$(3)))
	$(eval repo := $(if $(4),$(1)/$(4),$(1)))

	$(DOCKER_RUN) -i $(CALICO_BUILD) sh -c '\
		if [ ! -z "$(new_ver)" ]; then \
			$(GIT_CONFIG_SSH) \
			go get -d $(repo)@$(new_ver); \
			go mod tidy; \
		fi'
endef

# update_replace_pin updates the given package's version to the latest available in the specified repo and branch.
# This routine can only be used for packages being replaced in go.mod, such as private versions of open-source packages.
# $(1) should be the name of the package, $(2) and $(3) the repository and branch from which to update it. If $(4) is
# specified it's treated as the module version and use in the go mod edit -replace command.
define update_replace_pin
	$(eval new_ver := $(call get_remote_version,$(2),$(3)))
	$(eval original_repo := $(if $(4),$(1)/$(4),$(1)))
	$(eval replace_repo := $(if $(4),$(2)/$(4),$(2)))

	$(DOCKER_RUN) -i $(CALICO_BUILD) sh -c '\
		if [ ! -z "$(new_ver)" ]; then \
			$(GIT_CONFIG_SSH) \
			go mod edit -replace $(original_repo)=$(replace_repo)@$(new_ver); \
			go mod tidy; \
		fi'
endef


API_BRANCH?=$(PIN_BRANCH)
API_REPO?=github.com/projectcalico/calico/api
BASE_API_REPO?=github.com/projectcalico/calico/api
APISERVER_BRANCH?=$(PIN_BRANCH)
APISERVER_REPO?=github.com/projectcalico/calico/apiserver
TYPHA_BRANCH?=$(PIN_BRANCH)
TYPHA_REPO?=github.com/projectcalico/calico/typha
LIBCALICO_BRANCH?=$(PIN_BRANCH)
LIBCALICO_REPO?=github.com/projectcalico/calico/libcalico-go
CONFD_BRANCH?=$(PIN_BRANCH)
CONFD_REPO?=github.com/projectcalico/calico/confd
FELIX_BRANCH?=$(PIN_BRANCH)
FELIX_REPO?=github.com/projectcalico/calico/felix
CNI_BRANCH?=$(PIN_BRANCH)
CNI_REPO?=github.com/projectcalico/calico/cni-plugin

update-api-pin:
	$(call update_pin,$(API_REPO),$(API_REPO),$(API_BRANCH))

replace-api-pin:
	$(call update_replace_pin,$(BASE_API_REPO),$(API_REPO),$(API_BRANCH))

update-apiserver-pin:
	$(call update_pin,github.com/projectcalico/calico/apiserver,$(APISERVER_REPO),$(APISERVER_BRANCH))

replace-apiserver-pin:
	$(call update_replace_pin,github.com/projectcalico/calico/apiserver,$(APISERVER_REPO),$(APISERVER_BRANCH))

update-typha-pin:
	$(call update_pin,github.com/projectcalico/calico/typha,$(TYPHA_REPO),$(TYPHA_BRANCH))

replace-typha-pin:
	$(call update_replace_pin,github.com/projectcalico/calico/typha,$(TYPHA_REPO),$(TYPHA_BRANCH))

update-libcalico-pin:
	$(call update_pin,github.com/projectcalico/calico/libcalico-go,$(LIBCALICO_REPO),$(LIBCALICO_BRANCH))

replace-libcalico-pin:
	$(call update_replace_pin,github.com/projectcalico/calico/libcalico-go,$(LIBCALICO_REPO),$(LIBCALICO_BRANCH))

update-confd-pin:
	$(call update_replace_pin,github.com/kelseyhightower/confd,$(CONFD_REPO),$(CONFD_BRANCH))

update-felix-pin:
	$(call update_pin,github.com/projectcalico/calico/felix,$(FELIX_REPO),$(FELIX_BRANCH))

replace-felix-pin:
	$(call update_replace_pin,github.com/projectcalico/calico/felix,$(FELIX_REPO),$(FELIX_BRANCH))

update-cni-plugin-pin:
	$(call update_pin,github.com/projectcalico/calico/cni-plugin,$(CNI_REPO),$(CNI_BRANCH))

replace-cni-pin:
	$(call update_replace_pin,github.com/projectcalico/calico/cni-plugin,$(CNI_REPO),$(CNI_BRANCH))


git-status:
	git status --porcelain

git-config:
ifdef CONFIRM
	git config --global user.name "marvin-tigera"
	git config --global user.email "marvin@projectcalico.io"
endif

git-commit:
	git diff --quiet HEAD || git commit -m "Semaphore Automatic Update" go.mod go.sum $(EXTRA_FILES_TO_COMMIT)

###############################################################################
# External resource affecting macros
# The following macros affect resources outside of the local environment that
# they're run in, i.e. pushing to docker or github. If CONFIM is not defined,
# then the commands are just printed, instead of run.
#
# The <command>-cmd macro should never be run directly, it's used to define
# the command the macro runs but depending on whether CONFIRM is defined the
# command may be printed or run.
#
# You can redefine <command>-cmd to have the targets in this makefile use a
# different implementation.
###############################################################################

ifdef LOCAL_CRANE
CRANE_CMD         = crane
else
CRANE_CMD         = $(REPO_ROOT)/bin/crane
endif

ifdef LOCAL_PYTHON
PYTHON3_CMD       = python3
else
PYTHON3_CMD       = docker run --rm -e QUAY_API_TOKEN -v $(REPO_ROOT):$(REPO_ROOT) -w $(REPO_ROOT) python:3.13 python3.13
endif

GIT_CMD           = git
DOCKER_CMD        = docker


# RELEASE_PY3 is for Python invocations used for releasing,
# where we want to be able to DRY_RUN them.
ifdef CONFIRM
CRANE         = $(CRANE_CMD)
GIT           = $(GIT_CMD)
DOCKER        = $(DOCKER_CMD)
RELEASE_PY3   = $(PYTHON3_CMD)
else
CRANE         = echo [DRY RUN] $(CRANE_CMD)
GIT           = echo [DRY RUN] $(GIT_CMD)
DOCKER        = echo [DRY RUN] $(DOCKER_CMD)
RELEASE_PY3   = echo [DRY RUN] $(PYTHON3_CMD)
endif

QUAY_SET_EXPIRY_SCRIPT = $(REPO_ROOT)/hack/set_quay_expiry.py

commit-and-push-pr:
	$(GIT) add $(GIT_COMMIT_FILES)
	$(GIT) commit -m $(GIT_COMMIT_MESSAGE)
	$(GIT) push $(GIT_REMOTE) $(GIT_PR_BRANCH_HEAD)

###############################################################################
# GitHub API helpers
#   Helper macros and targets to help with communicating with the github API
###############################################################################
GIT_COMMIT_MESSAGE?="Automatic Pin Updates"
GIT_COMMIT_TITLE?="Semaphore Auto Pin Update"
GIT_PR_BRANCH_BASE?=$(SEMAPHORE_GIT_BRANCH)
PIN_UPDATE_BRANCH?=semaphore-auto-pin-updates-$(GIT_PR_BRANCH_BASE)
GIT_PR_BRANCH_HEAD?=$(PIN_UPDATE_BRANCH)
GIT_PIN_UPDATE_COMMIT_FILES?=go.mod go.sum
GIT_PIN_UPDATE_COMMIT_EXTRA_FILES?=$(GIT_COMMIT_EXTRA_FILES)
GIT_COMMIT_FILES?=$(GIT_PIN_UPDATE_COMMIT_FILES) $(GIT_PIN_UPDATE_COMMIT_EXTRA_FILES)

# Call the github API. $(1) is the http method type for the https request, $(2) is the repo slug, and is $(3) is for json
# data (if omitted then no data is set for the request). If GITHUB_API_EXIT_ON_FAILURE is set then the macro exits with 1
# on failure. On success, the ENV variable GITHUB_API_RESPONSE will contain the response from github
define github_call_api
	$(eval CMD := curl -f -X$(1) \
		-H "Content-Type: application/json"\
		-H "Authorization: token ${GITHUB_TOKEN}"\
		https://api.github.com/repos/$(2) $(if $(3),--data '$(3)',))
	$(eval GITHUB_API_RESPONSE := $(shell $(CMD) | sed -e 's/#/\\\#/g'))
	$(if $(GITHUB_API_EXIT_ON_FAILURE), $(if $(GITHUB_API_RESPONSE),,exit 1),)
endef

# Create the pull request. $(1) is the repo slug, $(2) is the title, $(3) is the head branch and $(4) is the base branch.
# If the call was successful then the ENV variable PR_NUMBER will contain the pull request number of the created pull request.
define github_pr_create
	$(eval JSON := {"title": "$(2)", "head": "$(3)", "base": "$(4)"})
	$(call github_call_api,POST,$(1)/pulls,$(JSON))
	$(eval PR_NUMBER := $(filter-out null,$(shell echo '$(GITHUB_API_RESPONSE)' | jq '.number')))
endef

# Create a comment on a pull request. $(1) is the repo slug, $(2) is the pull request number, and $(3) is the comment
# body.
define github_pr_add_comment
	$(eval JSON := {"body":"$(3)"})
	$(call github_call_api,POST,$(1)/issues/$(2)/comments,$(JSON))
endef

# List pull open pull requests for a head and base. $(1) is the repo slug, $(2) is the branch head, $(3) is the branch base,
# and $(4) is the state.
define github_pr_list
	$(eval QUERY := $(if $(2),head=$(2),)$(if $(3),\&base=$(3))$(if $(4),\&state=$(4),))
	$(call github_call_api,GET,$(1)/pulls?$(QUERY),)
endef

# Check if there is a pull request with head GIT_PR_BRANCH_HEAD and base GIT_PR_BRANCH_BASE for the repo with slug
# GIT_REPO_SLUG. If there is a PR that exists the PR_EXISTS will be set to 0, otherwise it is set to 1.
check-if-pin-update-pr-exists:
ifndef ORGANIZATION
	@echo "ORGANIZATION must be set for the project."
	exit 1
endif
	$(call github_pr_list,$(GIT_REPO_SLUG),$(ORGANIZATION):$(GIT_PR_BRANCH_HEAD),$(GIT_PR_BRANCH_BASE),open)
	$(eval PR_EXISTS := $(if $(filter-out 0,$(shell echo '$(GITHUB_API_RESPONSE)' | jq '. | length')),0,1))

###############################################################################
# Auto pin update targets
#   Targets updating the pins
###############################################################################
GITHUB_API_EXIT_ON_FAILURE?=1

## Update dependency pins to their latest changeset, committing and pushing it.
## DEPRECATED This will be removed along with associated helper functions in future releases. Use the trigger-auto-pin-update-process
## to create PR with the pin updates.
.PHONY: commit-pin-updates
commit-pin-updates: update-pins git-status git-config git-commit ci git-push

# Creates and checks out the branch defined by GIT_PR_BRANCH_HEAD. It attempts to delete the branch from the local and
# remote repositories. Requires CONFIRM to be set, otherwise it fails with an error.
create-pin-update-head: var-require-one-of-CONFIRM-DRYRUN
ifeq ($(shell git rev-parse --abbrev-ref HEAD),$(GIT_PR_BRANCH_HEAD))
	@echo "Current branch is pull request head, cannot set it up."
	exit 1
endif
	-git branch -D $(GIT_PR_BRANCH_HEAD)
	-$(GIT) push $(GIT_REMOTE) --delete $(GIT_PR_BRANCH_HEAD)
	git checkout -b $(GIT_PR_BRANCH_HEAD)

create-pin-update-pr:
	$(call github_pr_create,$(GIT_REPO_SLUG),[$(GIT_PR_BRANCH_BASE)] $(GIT_COMMIT_TITLE),$(GIT_PR_BRANCH_HEAD),$(GIT_PR_BRANCH_BASE))
	echo 'Created pin update pull request $(PR_NUMBER)'

# Add the "/merge-when-ready" comment to enable the "merge when ready" functionality, i.e. when the pull request is passing
# the tests and approved merge it. The PR_NUMBER is set by the dependent target
set-merge-when-ready-on-pin-update-pr:
	$(call github_pr_add_comment,$(GIT_REPO_SLUG),$(PR_NUMBER),/merge-when-ready delete-branch)
	echo "Added '/merge-when-ready' comment command to pull request $(PR_NUMBER)"

# Call the update-pins target with the GIT_PR_BRANCH_BASE as the PIN_BRANCH
trigger-pin-updates:
	PIN_BRANCH=$(GIT_PR_BRANCH_BASE) $(MAKE) update-pins

# POST_PIN_UPDATE_TARGETS is used to specify targets that should be run after the pins have been updated to run targets
# that modify files that are tied to the dependencies. An example would be generated files that would changed based on
# a dependency update. This target would likely need to be used in tandem with GIT_PIN_UPDATE_COMMIT_EXTRA_FILES so the
# update files are committed with the pin update.
POST_PIN_UPDATE_TARGETS ?=

# Trigger the auto pin update process. This involves updating the pins, committing and pushing them to github, creating
# a pull request, and add the "/merge-when-ready" comment command. If there is already a pin update PR for the base
# branch the pin update is not done and the target will exit.
trigger-auto-pin-update-process: check-if-pin-update-pr-exists
	$(if $(filter $(PR_EXISTS),0),echo "A pull request for head '$(GIT_PR_BRANCH_HEAD)' and base '$(GIT_PR_BRANCH_BASE)' already exists.",\
		$(MAKE) trigger-auto-pin-update-process-wrapped)

trigger-auto-pin-update-process-wrapped: create-pin-update-head trigger-pin-updates $(POST_PIN_UPDATE_TARGETS)
	$(if $(shell git diff --quiet HEAD $(GIT_COMMIT_FILES) || echo "true"),\
		$(MAKE) commit-and-push-pr create-pin-update-pr set-merge-when-ready-on-pin-update-pr,echo "Pins are up to date")

###############################################################################
# Static checks
#   repos can specify additional checks by setting LOCAL_CHECKS
###############################################################################
.PHONY: static-checks
## Run static source code checks (lint, formatting, ...)
static-checks: $(LOCAL_CHECKS)
	$(MAKE) golangci-lint

LINT_ARGS ?= --max-issues-per-linter 0 --max-same-issues 0 --timeout 15m

.PHONY: golangci-lint
golangci-lint: $(GENERATED_FILES)
	$(DOCKER_RUN) $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) golangci-lint run $(LINT_ARGS)'

REPO_REL_DIR=$(shell if [ -e hack/format-changed-files.sh ]; then echo '.'; else echo '..'; fi )

.PHONY: fix-changed go-fmt-changed goimports-changed
# Format changed files only.
fix-changed go-fmt-changed goimports-changed:
	if [ "$(SKIP_FIX_CHANGED)" != "true" ]; then \
	  parent_branch=`release_prefix=$(RELEASE_BRANCH_PREFIX)-v git_repo_slug=$(GIT_REPO_SLUG) $(REPO_REL_DIR)/hack/find-parent-release-branch.sh`; \
	  $(DOCKER_RUN) -e release_prefix=$(RELEASE_BRANCH_PREFIX)-v \
	                -e git_repo_slug=$(GIT_REPO_SLUG) \
	                -e parent_branch=$$parent_branch \
	                $(CALICO_BUILD) $(REPO_REL_DIR)/hack/format-changed-files.sh; \
	fi

.PHONY: fix-all go-fmt-all goimports-all
# $(DOCKER_GO_BUILD) is empty under NATIVE_GO_BUILD, so this runs the formatter directly.
fix-all go-fmt-all goimports-all:
	$(DOCKER_GO_BUILD) $(REPO_REL_DIR)/hack/format-all-files.sh

GOMODDER=$(REPO_REL_DIR)/hack/cmd/gomodder/main.go

.PHONY: verify-go-mods
verify-go-mods:
	# FIXME needs to be fixed up for calico-private repo.
	#$(DOCKER_RUN) $(CALICO_BUILD) go run $(GOMODDER)

.PHONY: pre-commit
pre-commit:
	$(DOCKER_RUN) $(CALICO_BUILD) git-hooks/pre-commit-in-container

.PHONY: install-git-hooks
install-git-hooks:
	${install_hooks}

.PHONY: check-module-path-tigera-api
check-module-path-tigera-api:
	@echo "Checking the repo importing tigera/api and not importing projectcalico/api"
	@IMPORT_TIGERA_API=$$($(DOCKER_GO_BUILD) sh -c 'go list -m github.com/tigera/api > /dev/null 2>&1 && echo yes || echo no'); \
	echo Is tigera/api imported? $$IMPORT_TIGERA_API; \
	if [ "$$IMPORT_TIGERA_API" != "yes" ]; then \
	     echo "Error: This repo should import tigera/api module."; \
	     false; \
	fi
	@IMPORT_PROJECTCALICO_API=$$($(DOCKER_GO_BUILD) sh -c 'go list -m github.com/projectcalico/calico/api > /dev/null 2>&1 && echo yes || echo no'); \
	echo Is projectcalico/api imported? $$IMPORT_PROJECTCALICO_API; \
	if [ "$$IMPORT_PROJECTCALICO_API" != "no" ]; then \
	     echo "Error: This repo should NOT import projectcalico/api module."; \
	     false; \
	fi

.PHONY: check-module-path-projectcalico-api
check-module-path-projectcalico-api:
	@echo "Checking the repo importing projectcalico/api and not importing tigera/api"
	@IMPORT_PROJECTCALICO_API=$$($(DOCKER_GO_BUILD) sh -c 'go list -m github.com/projectcalico/calico/api > /dev/null 2>&1 && echo yes || echo no'); \
	echo Is projectcalico/api imported? $$IMPORT_PROJECTCALICO_API; \
	if [ "$$IMPORT_PROJECTCALICO_API" != "yes" ]; then \
	     echo "Error: This repo should import projectcalico/api module."; \
	     false; \
	fi
	@IMPORT_TIGERA_API=$$($(DOCKER_GO_BUILD) sh -c 'go list -m github.com/tigera/api > /dev/null 2>&1 && echo yes || echo no'); \
	echo Is tigera/api imported? $$IMPORT_TIGERA_API; \
	if [ "$$IMPORT_TIGERA_API" != "no" ]; then \
	     echo "Error: This repo should NOT import tigera/api module."; \
	     false; \
	fi

###############################################################################
# go mod helpers
###############################################################################
mod-download:
	-$(DOCKER_RUN) $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) go mod download'

mod-tidy:
	-$(DOCKER_RUN) $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) go mod tidy'

###############################################################################
# Semaphore helpers
###############################################################################

# This semaphore project IDs are defined here because you cannot easily look them up in the semaphore API. This gives
# us a single place to define these values, then projects can reference the readable ENV variable when they need a semaphore
# project ID.
SEMAPHORE_API_PROJECT_ID=9625623e-bfc5-435f-9c22-74f9cd8622fc
SEMAPHORE_API_TIGERA_PROJECT_ID=48d23719-405f-4827-b58a-7de0598a6bf5
SEMAPHORE_API_SERVER_PROJECT_ID=6e4eb5b2-0150-4624-968d-f96a1cd9c37d
SEMAPHORE_API_SERVER_OSS_PROJECT_ID=10f6c7c1-7eaa-4e75-a9d1-83e5426158b1
SEMAPHORE_APP_POLICY_PRIVATE_PROJECT_ID=fa098f05-b2d2-4cf6-ac83-aa1e38e95670
SEMAPHORE_APP_POLICY_PROJECT_ID=bc654d5c-bb68-4b00-9d02-289291762b1d
SEMAPHORE_BIRD_PROJECT_ID=c1cc5eaf-873b-4113-a85e-a555361413e6
SEMAPHORE_CC_PORTAL=2b3f9721-a851-4a97-981f-0cb81f93ddd0
SEMAPHORE_CALICO_PRIVATE_PROJECT_ID=8a309869-f767-49dc-924f-fa927edbf657
SEMAPHORE_CALICO_PROJECT_ID=828e6de6-ed4b-49c7-9cb5-ac1246d454de
SEMAPHORE_CALICO_USAGE_PROJECT_ID=29f53c2b-8266-4873-879d-19b65960b3fd
SEMAPHORE_CALICOCTL_PRIVATE_PROJECT_ID=8d885379-6a1b-4fc8-aa45-dc0cfb87894a
SEMAPHORE_CALICOCTL_PROJECT_ID=193ce75a-7a47-4c9f-b966-f25c83e62213
SEMAPHORE_CALICOQ_PROJECT_ID=dc79e0e9-a7b3-40f5-8dc2-2818210ee0a9
SEMAPHORE_CLOUD_CONTROLLERS_PRIVATE_PROJECT_ID=f70e6c08-887b-481d-9591-68e243b32b32
SEMAPHORE_CNI_PLUGIN_PRIVATE_PROJECT_ID=f2c02a84-5fcd-49ed-b4cb-a6273409f0de
SEMAPHORE_CNI_PLUGIN_PROJECT_ID=741ec781-5dbb-4494-ba90-ec6831a9b176
SEMAPHORE_COMPLIANCE_PROJECT_ID=958a9147-ec94-4e99-b4c8-de7857653bb9
SEMAPHORE_CONFD_PROJECT_ID=4c6b815f-d42c-4436-aafa-651fbaf5859e
SEMAPHORE_CONFD_PRIVATE_PROJECT_ID=d3a7649a-3a39-45bf-95e9-fd6df3d0a7b1
SEMAPHORE_CURATOR_PROJECT_ID=c391dcff-6933-40e7-a6d1-1dcf7e6e231d
SEMAPHORE_DEEP_PACKET_INSPECTION_PROJECT_ID=81c0981e-979c-4741-8143-22166384afa1
SEMAPHORE_DEXIDP_DOCKER_PROJECT_ID=ee618372-35c8-4f83-bd05-d3a96ac2b276
SEMAPHORE_EGRESS_GATEWAY_PROJECT_ID=f01056ec-75f9-46a0-9ae2-6fc5e391136c
SEMAPHORE_ELASTICSEARCH_DOCKER_PROJECT_ID=0a3a5bf6-19e4-4210-a3fa-15fc857596ac
SEMAPHORE_ELASTICSEARCH_METRICS_PROJECT_ID=306b29c0-aa86-4b76-9c3e-c78a327e7d83
SEMAPHORE_ENVOY_DOCKER_PROJECT_ID=b8db000b-c2c4-44cd-a22d-51df73dfdcba
SEMAPHORE_ES_PROXY_IMAGE_PROJECT_ID=bc7ee48d-0051-4ceb-961d-03659463ada4
SEMAPHORE_ES_GATEWAY_PROJECT_ID=3c01c819-532b-4ccc-8305-5dd45c10bf93
SEMAPHORE_FELIX_PRIVATE_PROJECT_ID=e439cca4-156c-4d23-b611-002601440ad0
SEMAPHORE_FELIX_PROJECT_ID=48267e65-4acc-4f27-a88f-c3df0e8e2c3b
SEMAPHORE_FIREWALL_INTEGRATION_PROJECT_ID=d4307a31-1e46-4622-82e2-886165b77008
SEMAPHORE_INGRESS_COLLECTOR_PROJECT_ID=cf7947e4-a886-404d-ac6a-c3f3ac1a7b93
SEMAPHORE_INTRUSION_DETECTION_PROJECT_ID=2beffe81-b05a-41e0-90ce-e0d847dee2ee
SEMAPHORE_KEY_CERT_PROVISIONER_PROJECT_ID=9efb25f3-8c5d-4f22-aab5-4a1f5519bc7c
SEMAPHORE_KUBE_CONTROLLERS_PRIVATE_PROJECT_ID=0b8651d0-6c5d-4076-ab1d-25b120d0f670
SEMAPHORE_KUBE_CONTROLLERS_PROJECT_ID=d688e2ce-8c4a-4402-ba54-3aaa0eb53e5e
SEMAPHORE_KUBECTL_CALICO_PROJECT_ID=37d7cb2b-62b0-4178-9424-de766f2de59b
SEMAPHORE_KIBANA_DOCKER_PROJECT_ID=eaafdbad-4546-4582-b8fa-cea05a80a04d
SEMAPHORE_LIBCALICO_GO_PRIVATE_PROJECT_ID=72fa12b5-5ad5-43ae-b0ac-17f9f7c71030
SEMAPHORE_LIBCALICO_GO_PROJECT_ID=ce3e6bed-1fb6-4501-80e5-2121a266a386
SEMAPHORE_LICENSING_PROJECT_ID=344f1cf0-0c3f-4fa3-b89b-3c35127b3054
SEMAPHORE_L7_COLLECTOR_PROJECT_ID=b02e7bbf-39ee-4c0c-a6f6-793cdf89daa7
SEMAPHORE_LMA_PROJECT_ID=5130e1d3-d9cd-4270-9e62-57f98d34495e
SEMAPHORE_MANAGER_PROJECT_ID=325ca49d-5111-4b07-a54f-dc0c7ec538bb
SEMAPHORE_NETWORKING_CALICO_PROJECT_ID=0a7883cb-b727-4113-948d-b95cb00df6b6
SEMAPHORE_NODE_PRIVATE_PROJECT_ID=edd8246c-7116-473a-81c8-7a3bbbc07228
SEMAPHORE_NODE_PROJECT_ID=980a06a4-9d43-43f8-aedd-a3bfad258de6
SEMAPHORE_OPERATOR_PROJECT_ID=8343e619-cc44-4be4-a9d7-21963ebc1c8f
SEMAPHORE_PACKETCAPTURE_API_PROJECT_ID=f505b00c-57c3-4859-8b97-ff4095b5ab25
SEMAPHORE_PERFORMANCE_HOTSPOTS_PROJECT_ID=6a343a02-0acf-4c52-9cc7-24ee51377e32
SEMAPHORE_POD2DAEMON_PROJECT_ID=eb2eea4f-c185-408e-9837-da0d231428fb
SEMAPHORE_PROMETHEUS_SERVICE_PROJECT_ID=d5b7ed99-8966-46cc-90f2-9027c428db48
SEMAPHORE_SKIMBLE_PROJECT_ID=35171baf-8daf-4725-882f-c301851a6e1d
SEMAPHORE_TS_QUERYSERVER_PROJECT_ID=5dbe4688-0c21-40fb-89f7-a2d64c17401b
SEMAPHORE_TYPHA_PROJECT_ID=c2ea3f0a-58a0-427a-9ed5-6eff8d6543b3
SEMAPHORE_TYPHA_PRIVATE_PROJECT_ID=51e84cb9-0f38-408a-a113-0f5ca71844d7
SEMAPHORE_VOLTRON_PROJECT_ID=9d239362-9594-4c84-8983-868ee19ebd41

SEMAPHORE_WORKFLOW_BRANCH?=master

# Sends a request to the semaphore API to run the request workflow. It requires setting the SEMAPHORE_API_TOKEN, SEMAPHORE_PROJECT_ID,
# SEMAPHORE_WORKFLOW_BRANCH, and SEMAPHORE_WORKFLOW_FILE ENV variables.
semaphore-run-workflow:
	$(eval CMD := curl -f -X POST \
		-H "Authorization: Token $(SEMAPHORE_API_TOKEN)" \
		-d "project_id=$(SEMAPHORE_PROJECT_ID)&reference=$(SEMAPHORE_WORKFLOW_BRANCH)&commit_sha=$(SEMAPHORE_COMMIT_SHA)&pipeline_file=.semaphore/$(SEMAPHORE_WORKFLOW_FILE)" \
		"https://tigera.semaphoreci.com/api/v1alpha/plumber-workflows")
	$(eval SEMAPHORE_API_RESPONSE := $(shell $(CMD) | jq -R '.' | sed -e 's/#/\\\#/g'))
	$(if $(SEMAPHORE_API_RESPONSE),,exit 1)
	$(eval WORKFLOW_ID := $(shell echo $(SEMAPHORE_API_RESPONSE) | jq -r '.workflow_id'))
	@echo Semaphore workflow successfully created here https://tigera.semaphoreci.com/workflows/$(WORKFLOW_ID)

# This is a helpful wrapper of the semaphore-run-workflow target to run the update_pins workflow file for a project.
semaphore-run-auto-pin-update-workflow:
	SEMAPHORE_WORKFLOW_FILE=update_pins.yml $(MAKE) semaphore-run-workflow
	@echo Successfully triggered the semaphore pin update workflow

# This target triggers the 'semaphore-run-auto-pin-update-workflow' target for every SEMAPHORE_PROJECT_ID in the list of
# SEMAPHORE_AUTO_PIN_UPDATE_PROJECT_IDS.
semaphore-run-auto-pin-update-workflows:
	for ID in $(SEMAPHORE_AUTO_PIN_UPDATE_PROJECT_IDS); do\
		SEMAPHORE_WORKFLOW_BRANCH=$(SEMAPHORE_GIT_BRANCH) SEMAPHORE_PROJECT_ID=$$ID $(MAKE) semaphore-run-auto-pin-update-workflow; \
	done

###############################################################################
# Mock helpers
###############################################################################
# Helper targets for testify mock generation

# Generate testify mocks in the build container.
gen-mocks:
	$(DOCKER_RUN) $(CALICO_BUILD) sh -c '$(MAKE) mockery-run'
	# The generated files need import reordering to pass static-checks
	$(MAKE) fix-changed

# Run mockery (v3) to generate mocks using .mockery.yaml config.
# Look here for more information https://github.com/vektra/mockery
mockery-run:
	if [ -f .mockery.yaml ] || [ -f .mockery.yml ]; then \
		mockery; \
	else \
		echo "No .mockery.yaml found, skipping mock generation"; \
	fi

###############################################################################
# Docker helpers
###############################################################################
# Helper targets working with docker images.

# docker-compress takes the docker image specified by IMAGE_NAME and compresses all the layers into a single one. This is
# done by exporting the given image then re importing it with the given IMAGE_NAME.
#
# When a docker image is exported all of the instructions are lost (i.e. ENTRYPOINT, ENV, ...), so before the image is
# compressed the target inspects the image and pulls out the instructions. Each instruction that is pulled out is converted
# into a change directive, or change directives, of the format "--change 'INSTRUCTION <instruction>". These directives
# are given to the docker import command so the instructions can be re added to the compressed image.
#
# NOTE: This target does not attempt to copy every instruction from the original image to the compressed one. Any user of
# this target should ensure that any required instructions are copied over by this target.
docker-compress:
	$(eval JSONOBJ := "$(shell docker inspect $(IMAGE_NAME) | jq '.[0].Config' | jq -R '.' | sed -e 's/#/\\\#/g' ) ")
#	Re add the entry point.
	$(eval CHANGE := $(CHANGE)$(shell echo $(JSONOBJ) | jq -r \
		"if has(\"Entrypoint\") and .Entrypoint != \"\" then \" --change 'ENTRYPOINT \(.Entrypoint)'\" else \"\" end"\
	))
#	Re add the command.
	$(eval CHANGE := $(CHANGE)$(shell echo $(JSONOBJ) | jq -r \
		"if has(\"Cmd\") and .Cmd != \"\" then \" --change 'CMD \(.Cmd)'\" else \"\" end"\
	))
#	Re add the working directory.
	$(eval CHANGE := $(CHANGE)$(shell echo $(JSONOBJ) | jq -r \
		"if has(\"WorkingDir\") and .WorkingDir != \"\" then \" --change 'WORKDIR \(.WorkingDir)'\" else \"\" end"\
	))
#	Re add the user.
	$(eval CHANGE := $(CHANGE)$(shell echo $(JSONOBJ) | jq -r \
		"if has(\"User\") and .User != \"\" then \" --change 'USER \(.User)'\" else \"\" end"\
	))
#	Re add the environment variables. .Env is an array of strings so add a "--change 'ENV <value>'" for each value in
#	the array.
	$(eval CHANGE := $(CHANGE)$(shell echo $(JSONOBJ) | jq -r \
		"if has(\"Env\") and (.Env | length) > 0 then .Env | map(\" --change 'ENV \(.)'\") | join(\"\") else \"\" end"\
	))
#	Re add the labels. .Labels is a map of label names to label values, so add a "--change 'LABEL <key> <value>'" for
#	each map entry.
	$(eval CHANGE := $(CHANGE)$(shell echo $(JSONOBJ) | jq -r \
		"if has(\"Labels\") and (.Labels | length) > 0 then .Labels | to_entries | map(\" --change 'LABEL \(.key) \(.value)'\") | join(\"\") else \"\" end"\
	))
#	Re add the exposed ports. .ExposedPorts is a map, but we're only interested in the keys of the map so for each key
#	add "--change EXPOSE <key>".
	$(eval CHANGE := $(CHANGE)$(shell echo $(JSONOBJ) | jq -r \
		"if has(\"ExposedPorts\") and (.ExposedPorts | length) > 0 then .ExposedPorts | keys | map(\" --change 'EXPOSE \(.)'\") | join(\"\") else \"\" end"\
	))
	$(eval CONTAINER_ID := $(shell docker run -d -it --entrypoint /bin/true $(IMAGE_NAME) /bin/true))
	docker export $(CONTAINER_ID) | docker import $(CHANGE) - $(IMAGE_NAME)

###############################################################################
# Image building and pushing
###############################################################################

###############################################################################
# we want to be able to run the same recipe on multiple targets keyed on the image name
# to do that, we would use the entire image name, e.g. calico/node:abcdefg, as the stem, or '%', in the target
# however, make does **not** allow the usage of invalid filename characters - like / and : - in a stem, and thus errors out
# to get around that, we "escape" those characters by converting all : to --- and all / to ___ , so that we can use them
# in the target, we then unescape them back
escapefs = $(subst :,---,$(subst /,___,$(1)))
unescapefs = $(subst ---,:,$(subst ___,/,$(1)))

# retag-build-images-with-registries retags the build / arch images specified by BUILD_IMAGES and VALIDARCHES with
# the registries specified by DEV_REGISTRIES. The end tagged images are of the format
# $(REGISTRY)/$(BUILD_IMAGES):<tag>-$(ARCH).
retag-build-images-with-registries: $(addprefix retag-build-images-with-registry-,$(call escapefs,$(DEV_REGISTRIES)))

# retag-build-images-with-registry-% retags the build / arch images specified by BUILD_IMAGES and VALIDARCHES with
# the registry specified by $*.
retag-build-images-with-registry-%:
	$(MAKE) $(addprefix retag-build-image-with-registry-,$(call escapefs,$(BUILD_IMAGES))) REGISTRY=$(call unescapefs,$*)

# retag-build-image-with-registry-% retags the build arch images specified by $* and VALIDARCHES with the
# registry specified by REGISTRY.
retag-build-image-with-registry-%: var-require-all-REGISTRY-BUILD_IMAGES
	$(MAKE) $(addprefix retag-build-image-arch-with-registry-,$(VALIDARCHES)) BUILD_IMAGE=$(call unescapefs,$*)

# retag-build-image-arch-with-registry-% retags the build / arch image specified by $* and BUILD_IMAGE with the
# registry specified by REGISTRY. SOURCE_IMAGE lets a component decouple its
# local tag from BUILD_IMAGE - e.g. cmd/calico tags locally as `tigera/calico`
# but publishes as `<REGISTRY>/calico`.
SOURCE_IMAGE ?= $(BUILD_IMAGE)
retag-build-image-arch-with-registry-%: var-require-all-REGISTRY-BUILD_IMAGE-IMAGETAG
	docker tag $(SOURCE_IMAGE):$(LATEST_IMAGE_TAG)-$* $(call filter-registry,$(REGISTRY))$(BUILD_IMAGE):$(IMAGETAG)-$*
	$(if $(filter $*,amd64),\
		docker tag $(SOURCE_IMAGE):$(LATEST_IMAGE_TAG)-$(ARCH) $(REGISTRY)/$(BUILD_IMAGE):$(IMAGETAG),\
		$(NOECHO) $(NOOP)\
	)

# validate-dev-registries fails if any entry in DEV_REGISTRIES is not prefixed
# by an entry in ALLOWED_DEV_REGISTRIES. Intended to prevent accidental pushes
# to registries outside Tigera control - notably the OSS quay.io/calico and
# docker.io/calico paths owned by projectcalico/calico, which previously got
# clobbered by an Enterprise CI job.
#
# Implemented purely with make string functions (no shell loop) so it behaves
# identically under sh and powershell.exe. The third-party base image builds
# (e.g. third_party/fluentd-base) set SHELL := powershell.exe for their Windows
# publish, which cannot parse a Bourne-shell if/for/case recipe. A registry r
# is allowed when it starts with some prefix in ALLOWED_DEV_REGISTRIES, i.e.
# $(filter <prefix>%,r) is non-empty. On a violation $(error) aborts make
# directly - the shell is never invoked; otherwise the recipe expands to
# nothing and is skipped.
DISALLOWED_DEV_REGISTRIES = $(foreach r,$(DEV_REGISTRIES),$(if $(strip $(foreach a,$(ALLOWED_DEV_REGISTRIES),$(filter $(a)%,$(r)))),,$(r)))
validate-dev-registries:
	$(if $(strip $(ALLOWED_DEV_REGISTRIES)),$(if $(strip $(DISALLOWED_DEV_REGISTRIES)),$(error DEV_REGISTRIES entry "$(strip $(DISALLOWED_DEV_REGISTRIES))" is not prefixed by any value in ALLOWED_DEV_REGISTRIES ("$(ALLOWED_DEV_REGISTRIES)") -- update ALLOWED_DEV_REGISTRIES in metadata.mk if this push target is now legitimate)))

# push-images-to-registries pushes the build / arch images specified by BUILD_IMAGES and VALIDARCHES to the registries
# specified by DEV_REGISTRY.
push-images-to-registries: validate-dev-registries $(addprefix push-images-to-registry-,$(call escapefs,$(DEV_REGISTRIES)))

# push-images-to-registry-% pushes the build / arch images specified by BUILD_IMAGES and VALIDARCHES to the registry
# specified by %*.
push-images-to-registry-%:
	$(MAKE) $(addprefix push-image-to-registry-,$(call escapefs,$(BUILD_IMAGES))) REGISTRY=$(call unescapefs,$*)

# push-image-to-registry-% pushes the build / arch images specified by $* and VALIDARCHES to the registry
# specified by REGISTRY.
#
# We also build a list of all of the iamges we're going to be pushing and then, once we're done, we set the expiry
# on those images, either adding an expiry for normal pushes or removing it for releases, if we're pushing to quay.io
# that is.
push-image-to-registry-%:
	$(eval BUILD_IMAGE=$(call unescapefs,$*))
	$(eval EXPIRY_IMAGES_LIST=$(REGISTRY)/$(BUILD_IMAGE):$(IMAGETAG) $(addprefix $(REGISTRY)/$(BUILD_IMAGE):$(IMAGETAG)-,$(VALIDARCHES)))

	$(MAKE) -j6 $(addprefix push-image-arch-to-registry-,$(VALIDARCHES)) BUILD_IMAGE=$(BUILD_IMAGE)

ifeq ($(findstring quay.io,$(REGISTRY)),quay.io)
	$(if $(RELEASE), \
		-$(RELEASE_PY3) $(QUAY_SET_EXPIRY_SCRIPT) remove $(EXPIRY_IMAGES_LIST), \
		-$(RELEASE_PY3) $(QUAY_SET_EXPIRY_SCRIPT) add --expiry-days=$(QUAY_EXPIRE_DAYS) $(EXPIRY_IMAGES_LIST) \
	)
endif

# retry_docker_cmd retries a docker command up to a specified number of times.
# Usage: $(call retry_docker_cmd,<description>,<docker command>,<max retries>,<retry delay>)
#
# Windows agents (Semaphore) execute recipe lines under PowerShell, which can't parse
# the bash retry loop used elsewhere; emit a single-line PowerShell loop instead.
ifeq ($(OS),Windows_NT)
define retry_docker_cmd
	for ($$i=1; $$i -le $(3); $$i++) { $(2); if ($$LASTEXITCODE -eq 0) { break }; Write-Host ('WARNING: $(1) failed (attempt {0}/$(3)), retrying in $(4)s...' -f $$i); if ($$i -eq $(3)) { exit 1 }; Start-Sleep -Seconds $(4) }
endef
else
define retry_docker_cmd
	i=1; \
	while [ $$i -le $(3) ]; do \
		$(2) && break; \
		echo "WARNING: $(1) failed (attempt $$i/$(3)), retrying in $(4)s..."; \
		if [ $$i -eq $(3) ]; then exit 1; fi; \
		sleep $(4); \
		i=$$((i + 1)); \
	done
endef
endif

# Configuration options for retrying docker commands
MANIFEST_RETRIES ?= 5
MANIFEST_RETRY_DELAY ?= 5

# log_step prints a grep-able timing record. Bracket a long-running shell
# step with start/end calls so per-step durations can be computed from the
# CI log without per-line instrumentation.
# Usage: $(call log_step,start,<label>)  ...  $(call log_step,end,<label>)
define log_step
	echo "==> step=$(1) label=\"$(2)\" epoch=$$(date +%s) at=$$(date -u +%FT%TZ)"
endef

# push-image-arch-to-registry-% pushes the build / arch image specified by $* and BUILD_IMAGE to the registry
# specified by REGISTRY.
push-image-arch-to-registry-%:
# If the registry we want to push to doesn't not support manifests don't push the ARCH image.
	@$(call log_step,start,push $(call filter-registry,$(REGISTRY))$(BUILD_IMAGE):$(IMAGETAG)-$*)
	$(call retry_docker_cmd,docker push with quiet flag,$(DOCKER) push --quiet $(call filter-registry,$(REGISTRY))$(BUILD_IMAGE):$(IMAGETAG)-$*,$(MANIFEST_RETRIES),$(MANIFEST_RETRY_DELAY))
	@$(call log_step,end,push $(call filter-registry,$(REGISTRY))$(BUILD_IMAGE):$(IMAGETAG)-$*)
	$(if $(filter $*,amd64),\
		$(call log_step,start,push $(REGISTRY)/$(BUILD_IMAGE):$(IMAGETAG)); \
		$(call retry_docker_cmd,docker push,$(DOCKER) push $(REGISTRY)/$(BUILD_IMAGE):$(IMAGETAG),$(MANIFEST_RETRIES),$(MANIFEST_RETRY_DELAY)); \
		$(call log_step,end,push $(REGISTRY)/$(BUILD_IMAGE):$(IMAGETAG)) \
	)

# push multi-arch manifest where supported.
push-manifests: var-require-all-IMAGETAG validate-dev-registries $(addprefix sub-manifest-,$(call escapefs,$(PUSH_MANIFEST_IMAGES)))
sub-manifest-%:
	@if [ -z "$(MANIFEST_RETRIES)" ] || ! printf '%s\n' "$(MANIFEST_RETRIES)" | grep -Eq '^[0-9]+$$' || [ "$(MANIFEST_RETRIES)" -lt 1 ]; then \
		echo "ERROR: MANIFEST_RETRIES must be a positive integer, got '$(MANIFEST_RETRIES)'"; \
		exit 1; \
	fi
	@$(call log_step,start,manifest-create $(call unescapefs,$*):$(IMAGETAG))
	$(call retry_docker_cmd,docker manifest create,$(DOCKER) manifest create $(call unescapefs,$*):$(IMAGETAG) $(addprefix --amend ,$(addprefix $(call unescapefs,$*):$(IMAGETAG)-,$(VALIDARCHES))),$(MANIFEST_RETRIES),$(MANIFEST_RETRY_DELAY))
	@$(call log_step,end,manifest-create $(call unescapefs,$*):$(IMAGETAG))
	@$(call log_step,start,manifest-push $(call unescapefs,$*):$(IMAGETAG))
	$(call retry_docker_cmd,docker manifest push,$(DOCKER) manifest push --purge $(call unescapefs,$*):$(IMAGETAG),$(MANIFEST_RETRIES),$(MANIFEST_RETRY_DELAY))
	@$(call log_step,end,manifest-push $(call unescapefs,$*):$(IMAGETAG))

push-manifests-with-tag: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME
	$(MAKE) push-manifests IMAGETAG=$(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(BRANCH_NAME) VALIDARCHES="$(VALIDARCHES)"
	$(MAKE) push-manifests IMAGETAG=$(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(GIT_VERSION) VALIDARCHES="$(VALIDARCHES)"

# cd-common tags and pushes images with the branch name and git version. This target uses PUSH_IMAGES, BUILD_IMAGES,
# and BRANCH_NAME env variables to figure out what to tag and where to push them to.
cd-common: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME
	$(MAKE) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(BRANCH_NAME) VALIDARCHES="$(VALIDARCHES)"
	$(MAKE) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(GIT_VERSION) VALIDARCHES="$(VALIDARCHES)"

###############################################################################
# Release targets and helpers
#
# The following targets and macros are used to help start and cut releases.
# At high level, this involves:
# - Creating release branches
# - Adding empty commits to start next release, and updating the 'dev' tag
# - Adding 'release' tag to the commit that will be release
# - Creating an empty commit for the next potential patch release, and updating
#   the dev tag on that commit
# - Copying images for the released commit over to the release registries, and
#   re tagging those images with the release tag
#
# The following definitions will be helpful in understanding this process:
# - 'dev' tag: A git tag of the form of `v3.8.0-calient-0.dev-36-g3a618e61c2d3`
#   that every commit has. The start of the dev tag, i.e. v3.8.0, is the
#   the release that this commit will go into.
# - 'release' tag: A git tag of the form of `v3.8.0`. The commit that a release
#   is cut from will have this tag, i.e. you can find the commit that release
#   3.8 uses by finding the commit with the tag v3.8.0.
# - 'dev' image: The image that is created for every commit that is merged to
#   master or a release branch. This image is tagged with the dev tag, i.e.
#   if commit 3a618e61c2d3 is on master or a release branch, there will be
#   an image for that commit in the dev registry with the tag
#   `v3.8.0-calient-0.dev-36-g3a618e61c2d3`.
# - 'release' image: The public image the customers will use to install our
#   our product. Producing this is the goal of cutting the release. This image
#   will be in the release registries, and will be tagged with the release tag,
#   i.e. the release image for release 3.8 will have the v3.8.0 tag, or if it's
#   a patch release it will be v3.8.<patch version>
###############################################################################
fetch-all:
	git fetch --all -q

# git-dev-tag retrieves the dev tag for the current commit (the one are dev images are tagged with).
git-dev-tag = $(shell git describe --tags --long --always --abbrev=12 --match "*dev*")
# git-release-tag-from-dev-tag gets the release version from the current commits dev tag.
git-release-tag-from-dev-tag = $(shell echo $(call git-dev-tag) | grep -P -o "^v\d*.\d*.\d*(-.*)?(?=-$(DEV_TAG_SUFFIX))")
# git-release-tag-for-current-commit gets the release tag for the current commit if there is one.
git-release-tag-for-current-commit = $(shell git describe --tags --exact-match --exclude "*dev*")

# release-branch-for-tag finds the latest branch that corresponds to the given tag.
release-branch-for-tag = $(firstword $(shell git --no-pager branch --format='%(refname:short)' --contains $1 | grep -P "^release"))
# commit-for-tag finds the latest commit that corresponds to the given tag.
commit-for-tag = $(shell git rev-list -n 1 $1)
git-commit-for-remote-tag = $(shell git ls-remote -q --tags $(GIT_REMOTE) $1 | awk '{print $$1}')
# current-branch gets the name of the branch for the current commit.
current-branch = $(shell git rev-parse --abbrev-ref HEAD)

# RELEASE_BRANCH_BASE is used when creating a release branch to confirm the correct base is being used. It's
# configurable so that a dry run can be done from a PR branch.
RELEASE_BRANCH_BASE ?=master

# var-set-% checks if there is a non empty variable for the value describe by %. If FAIL_NOT_SET is set, then var-set-%
# fails with an error message. If FAIL_NOT_SET is not set, then var-set-% appends a 1 to VARSET if the variable isn't
# set.
var-set-%:
	$(if $($*),$(eval VARSET+=1),$(if $(FAIL_NOT_SET),$(error $* is required but not set),))

# var-require is used to check if one or all of the variables are set in REQUIRED_VARS, and fails if not. The variables
# in REQUIRE_VARS are hyphen separated.
#
# If FAIL_NOT_SET is set, then all variables described in REQUIRED_VARS must be set for var-require to not fail,
# otherwise only one variable needs to be set for var-require to not fail.
var-require: $(addprefix var-set-,$(subst -, ,$(REQUIRED_VARS)))
	$(if $(VARSET),,$(error one of $(subst -, ,$(REQUIRED_VARS)) is not set or empty, but at least one is required))

# var-require-all-% checks if the there are non empty variables set for the hyphen separated values in %, and fails if
# there isn't a non empty variable for each given value. For instance, to require FOO and BAR both must be set you would
# call var-require-all-FOO-BAR.
var-require-all-%:
	@$(MAKE) --quiet --no-print-directory var-require REQUIRED_VARS=$* FAIL_NOT_SET=true

# var-require-one-of-% checks if the there are non empty variables set for the hyphen separated values in %, and fails
# there isn't a non empty variable for at least one of the given values. For instance, to require either FOO or BAR both
# must be set you would call var-require-all-FOO-BAR.
var-require-one-of-%:
	@$(MAKE) --quiet --no-print-directory var-require REQUIRED_VARS=$*

# build-images echos the images that would be built.
# If WINDOWS_IMAGE is set then it echos the windows image that would be built as well.
build-images: var-require-all-BUILD_IMAGES
	$(if $(WINDOWS_IMAGE),\
		@echo $(BUILD_IMAGES) $(WINDOWS_IMAGE),\
		@echo $(BUILD_IMAGES)\
	)

# image-tag-prefix echoes the prefix this component's published tags carry, which
# a variant selects through its own environment. Empty for an unprefixed build.
image-tag-prefix:
	@echo $(IMAGETAG_PREFIX)

# sem-cut-release triggers the cut-release pipeline (or test-cut-release if CONFIRM is not specified) in semaphore to
# cut the release. The pipeline is triggered for the current commit, and the branch it's triggered on is calculated
# from the RELEASE_VERSION, CNX, and OS variables given.
#
# Before the pipeline is triggered, this target validates that the expected release will be cut using the
# RELEASE_TAG (optional and defaults to the current tag) and RELEASE_VERSION (required) variables. The RELEASE_TAG
# should be the dev tag that the release is cut from, and RELEASE_VERSION should be the version expected to be released.
# This target verifies that the current commit is tagged with the RELEASE_TAG and that cutting this commit will result
# in RELEASE_VERSION being cut.
sem-cut-release: var-require-one-of-CONFIRM-DRYRUN var-require-all-RELEASE_VERSION var-require-one-of-CNX-OS
ifndef RELEASE_TAG
	$(eval RELEASE_TAG = $(call git-dev-tag))
else
	$(eval RELEASE_TAG_COMMIT = $(call commit-for-tag,$(RELEASE_TAG)))
	$(if $(filter-out $(RELEASE_TAG_COMMIT),$(GIT_COMMIT)),\
		echo Current commit is not tagged with $(RELEASE_TAG) && exit 1)
endif
	$(eval CURRENT_RELEASE_VERSION = $(call git-release-tag-from-dev-tag))
	$(if $(filter-out $(CURRENT_RELEASE_VERSION),$(RELEASE_VERSION)),\
		echo Given release version $(RELEASE_VERSION) does not match current commit release version $(CURRENT_RELEASE_VERSION). && exit 1)

	$(eval RELEASE_BRANCH = release-$(if $CNX,calient-,)$(shell echo "$(RELEASE_VERSION)" | awk -F  "." '{print $$1"."$$2}'))
	$(eval WORKFLOW_FILE = $(if $(CONFIRM),cut-release.yml,test-cut-release.yml))

	@echo Cutting release for $(RELEASE_VERSION) from dev tag $(RELEASE_TAG) \(commit $(GIT_COMMIT)\)
	SEMAPHORE_WORKFLOW_BRANCH=$(RELEASE_BRANCH) SEMAPHORE_COMMIT_SHA=$(GIT_COMMIT) SEMAPHORE_WORKFLOW_FILE=$(WORKFLOW_FILE) $(MAKE) semaphore-run-workflow

# cut-release uses the dev tags on the current commit to cut the release, more specifically cut-release does the
# following:
# - Calculates the release tag from the dev tag on the commit
# - tags the current commit with the release tag then pushes that tag to github
# - retags the build images (specified by BUILD_IMAGES) in the dev registries (specified DEV_REGISTRIES) with the
#	release tag
# - copies the build images (specified by BUILD_IMAGES) from the first dev registry to the release registries (specified
#	by RELEASE_REGISTRIES) and retags those images with the release tag
# - tags an empty commit at the head of the release branch with the next patch release dev tag and pushed that to github
cut-release: var-require-one-of-CONFIRM-DRYRUN
	$(MAKE) cut-release-wrapped RELEASE=true

cut-release-wrapped: var-require-one-of-CONFIRM-DRYRUN
	$(eval DEV_TAG = $(call git-dev-tag))
	$(eval RELEASE_TAG = $(call git-release-tag-from-dev-tag))
	$(eval RELEASE_BRANCH = $(call release-branch-for-tag,$(DEV_TAG)))
ifdef EXPECTED_RELEASE_TAG
	$(if $(filter-out $(RELEASE_TAG),$(EXPECTED_RELEASE_TAG)),\
		@echo "Failed to verify release tag$(comma) expected release version is $(EXPECTED_RELEASE_TAG)$(comma) actual is $(RELEASE_TAG)."\
		&& exit 1)
endif
	$(eval NEXT_RELEASE_VERSION := $(shell echo "$(call git-release-tag-from-dev-tag)" | awk '{ split($$0,tag,"-"); if (tag[2] ~ /^1\./) { split(tag[2],subver,"."); print tag[1]"-"subver[1]+1".0" } else { split(tag[1],ver,"."); print ver[1]"."ver[2]"."ver[3]+1 } }'))
ifndef IMAGE_ONLY
	$(MAKE) maybe-tag-release maybe-push-release-tag\
		RELEASE_TAG=$(RELEASE_TAG) BRANCH=$(RELEASE_BRANCH) DEV_TAG=$(DEV_TAG)
endif
ifdef BUILD_IMAGES
	$(eval IMAGE_DEV_TAG = $(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(DEV_TAG))
	$(eval IMAGE_RELEASE_TAG = $(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(RELEASE_TAG))
	$(MAKE) release-dev-images\
		RELEASE_TAG=$(IMAGE_RELEASE_TAG) BRANCH=$(RELEASE_BRANCH) DEV_TAG=$(IMAGE_DEV_TAG)
endif
ifndef IMAGE_ONLY
	$(MAKE) maybe-dev-tag-next-release maybe-push-next-release-dev-tag\
		NEXT_RELEASE_VERSION=$(NEXT_RELEASE_VERSION) BRANCH=$(RELEASE_BRANCH) DEV_TAG=$(DEV_TAG)
endif

cut-release-image: var-require-one-of-CONFIRM-DRYRUN
	$(eval DEV_TAG = $(if $(DEV_TAG),$(DEV_TAG),$(call git-dev-tag)))
	$(eval RELEASE_TAG = $(if $(RELEASE_TAG),$(RELEASE_TAG),$(call git-release-tag-from-dev-tag)))
	$(eval RELEASE_BRANCH = $(call release-branch-for-tag,$(DEV_TAG)))
	$(eval IMAGE_DEV_TAG = $(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(DEV_TAG))
	$(eval IMAGE_RELEASE_TAG = $(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(RELEASE_TAG))
ifdef BUILD_IMAGES
	$(MAKE) release-dev-images\
		RELEASE_TAG=$(IMAGE_RELEASE_TAG) BRANCH=$(RELEASE_BRANCH) DEV_TAG=$(IMAGE_DEV_TAG)
endif

# maybe-tag-release calls the tag-release target only if the current commit is not tagged with the tag in RELEASE_TAG.
# If the current commit is already tagged with the value in RELEASE_TAG then this is a NOOP.
maybe-tag-release: var-require-all-RELEASE_TAG
	$(if $(filter-out $(call git-release-tag-for-current-commit),$(RELEASE_TAG)),\
		$(MAKE) tag-release,\
		@echo "Current commit already tagged with $(RELEASE_TAG)")

# tag-release tags the current commit with an annotated tag with the value in RELEASE_TAG. This target throws an error
# if the current branch is not master.
tag-release: var-require-one-of-CONFIRM-DRYRUN var-require-all-DEV_TAG_SUFFIX-RELEASE_TAG
	$(if $(filter-out $(RELEASE_BRANCH_BASE),$(call current-branch)),,$(error tag-release cannot be called on $(RELEASE_BRANCH_BASE)))
	git tag -a $(RELEASE_TAG) -m "Release $(RELEASE_TAG)"

# maybe-push-release-tag calls the push-release-tag target only if the tag in RELEASE_TAG is not already pushed to
# github. If the tag is pushed to github then this is a NOOP.
# TODO should we check the commit tagged in remote is the current commit? Probably yes... that could catch some annoying problems that would be hard to find if they happened...
maybe-push-release-tag: var-require-all-RELEASE_TAG
	$(if $(shell git ls-remote -q --tags $(GIT_REMOTE) $(RELEASE_TAG)),\
		@echo Release $(RELEASE_TAG) already in github,\
		$(MAKE) push-release-tag)

# push-release-tag pushes the tag in RELEASE_TAG to github. If the current commit is not tagged with this tag then this
# target fails.
push-release-tag: var-require-one-of-CONFIRM-DRYRUN var-require-all-DEV_TAG_SUFFIX-RELEASE_TAG
	$(if $(call git-release-tag-for-current-commit),,$(error Commit does not have a release tag))
	$(GIT) push $(GIT_REMOTE) $(RELEASE_TAG)

# maybe-dev-tag-next-release calls the dev-tag-next-release-target only if the tag NEXT_RELEASE_VERSION-DEV_TAG_SUFFIX
# doesn't exist locally. If the tag does exist then this is a NOOP.
maybe-dev-tag-next-release: var-require-all-NEXT_RELEASE_VERSION-DEV_TAG_SUFFIX
	$(if $(shell git rev-parse --verify -q "$(NEXT_RELEASE_VERSION)-$(DEV_TAG_SUFFIX)"),\
		echo "Tag for next release $(NEXT_RELEASE_VERSION) already exists$(comma) not creating.",\
		$(MAKE) dev-tag-next-release)

# dev-tag-next-release creates a new commit empty commit at the head of BRANCH and tags it with
# NEXT_RELEASE_VERSION-DEV_TAG_SUFFIX.
dev-tag-next-release: var-require-one-of-CONFIRM-DRYRUN var-require-all-NEXT_RELEASE_VERSION-DEV_TAG_SUFFIX-BRANCH
	git checkout $(BRANCH)
	$(GIT) pull $(GIT_REMOTE) $(BRANCH)
	git commit --allow-empty -m "Begin development on $(NEXT_RELEASE_VERSION)"
	git tag $(NEXT_RELEASE_VERSION)-$(DEV_TAG_SUFFIX)

# maybe-push-next-release-dev-tag calls the push-next-release-dev-tag target if the tag
# NEXT_RELEASE_VERSION-DEV_TAG_SUFFIX doesn't exist remotely. If the tag exists remotely then this is a NOOP.
maybe-push-next-release-dev-tag: var-require-one-of-CONFIRM-DRYRUN var-require-all-NEXT_RELEASE_VERSION-DEV_TAG_SUFFIX
	$(if $(shell git ls-remote --tags $(GIT_REMOTE) $(NEXT_RELEASE_VERSION)-$(DEV_TAG_SUFFIX)),\
		echo "Dev tag for next release $(NEXT_RELEASE_VERSION) already pushed to github.",\
		$(MAKE) push-next-release-dev-tag)

# push-next-release-dev-tag pushes the tag NEXT_RELEASE_VERSION-DEV_TAG_SUFFIX and the current branch to github. If
# the current branch is not the head of the branch then this target fails.
push-next-release-dev-tag: var-require-one-of-CONFIRM-DRYRUN var-require-all-NEXT_RELEASE_VERSION-DEV_TAG_SUFFIX
	# The next release commit should always be at the head of a release branch.
	$(if $(filter-out HEAD,$(call current-branch)),,\
		$(error "Refusing to push commit for next release while in a detached state."))
	$(GIT) push $(GIT_REMOTE) $(call current-branch)
	$(GIT) push $(GIT_REMOTE) $(NEXT_RELEASE_VERSION)-$(DEV_TAG_SUFFIX)

# release-dev-images releases the dev images by calling the release-tag-dev-image-% and publish-dev-image-% on each
# value in BUILD_IMAGES. This results in retagging all the dev images with the release tag and copying the dev images
# over to the release registries.
ifndef SKIP_DEV_IMAGE_RETAG
RELEASE_DEV_IMAGES_RETAG_TARGETS ?= $(addprefix release-retag-dev-images-in-registry-,$(call escapefs, $(DEV_REGISTRIES)))
endif

RELEASE_DEV_IMAGES_TARGETS ?= $(addprefix release-dev-images-to-registry-,$(call escapefs, $(RELEASE_REGISTRIES)))
release-dev-images: var-require-one-of-CONFIRM-DRYRUN var-require-all-BUILD_IMAGES $(RELEASE_DEV_IMAGES_RETAG_TARGETS) $(RELEASE_DEV_IMAGES_TARGETS)

# release-retag-dev-images-in-registry-% retags all the build / arch images specified by BUILD_IMAGES and VALIDARCHES in
# the registry specified by $* with the release tag specified by RELEASE_TAG.
release-retag-dev-images-in-registry-%:
	$(MAKE) $(addprefix release-retag-dev-image-in-registry-,$(call escapefs, $(BUILD_IMAGES))) DEV_REGISTRY=$(call unescapefs,$*)

# release-retag-dev-image-in-registry-% retags the build image specified by $* in the dev registry specified by
# DEV_REGISTRY with the release tag specified by RELEASE_TAG. If DEV_REGISTRY is in the list of registries specified by
# RELEASE_REGISTRIES then the retag is not done
release-retag-dev-image-in-registry-%: bin/crane
	$(if $(filter-out $(RELEASE_REGISTRIES),$(DEV_REGISTRY)),\
		$(CRANE) cp $(DEV_REGISTRY)/$(call unescapefs,$*):$(DEV_TAG) $(DEV_REGISTRY)/$(call unescapefs,$*):$(RELEASE_TAG))

# release-dev-images-to-registry-% copies and retags all the build / arch images specified by BUILD_IMAGES and
# VALIDARCHES from the registry specified by DEV_REGISTRY to the registry specified by RELEASE_REGISTRY using the tag
# specified by DEV_TAG and RELEASE_TAG.
release-dev-images-to-registry-%:
	$(MAKE) $(addprefix release-dev-image-to-registry-,$(call escapefs, $(BUILD_IMAGES))) RELEASE_REGISTRY=$(call unescapefs,$*)

# release-dev-image-to-registry-% copies the build image and build arch images specified by $* and VALIDARCHES from
# the dev repo specified by DEV_TAG and RELEASE.
release-dev-image-to-registry-%: bin/crane
	$(if $(SKIP_MANIFEST_RELEASE),,\
		$(CRANE) cp $(DEV_REGISTRY)/$(call unescapefs,$*):$(DEV_TAG) $(RELEASE_REGISTRY)/$(call unescapefs,$*):$(RELEASE_TAG))
	$(if $(SKIP_ARCH_RELEASE),,\
		$(MAKE) $(addprefix release-dev-image-arch-to-registry-,$(VALIDARCHES)) BUILD_IMAGE=$(call unescapefs,$*))

# release-dev-image-to-registry-% copies the build arch image specified by BUILD_IMAGE and ARCH from the dev repo
# specified by DEV_TAG and RELEASE.
release-dev-image-arch-to-registry-%: bin/crane
	$(CRANE) cp $(DEV_REGISTRY)/$(BUILD_IMAGE):$(DEV_TAG)-$* $(RELEASE_REGISTRY)/$(BUILD_IMAGE):$(RELEASE_TAG)-$*

# release-prereqs checks that the environment is configured properly to create a release.
.PHONY: release-prereqs
release-prereqs:
ifndef VERSION
	$(error VERSION is undefined - run using make release VERSION=vX.Y.Z)
endif

# Check if the codebase is dirty or not.
check-dirty:
	@if [ "$$(git --no-pager diff --stat)" != "" ]; then \
	echo "The following files are dirty"; git --no-pager diff; exit 1; fi

.PHONY: bin/crane

# This setup is used to download and install the `crane` binary into $(REPO_ROOT)/bin/crane (or crane.exe for Windows).

.PHONY: bin/crane
CRANE_URL = https://github.com/google/go-containerregistry/releases/download/$(CRANE_VERSION)/go-containerregistry_$(CRANE_OS)_$(CRANE_ARCH).tar.gz

# Special case for Windows - we don't use this yet, it will need testing
ifeq ($(OS),Windows_NT)
CRANE_ARCH = x86_64
CRANE_CMD = $(CRANE_CMD).exe
CRANE_OS = Windows

# Install crane binary into bin/
bin/crane: $(REPO_ROOT)/bin/crane.exe
$(REPO_ROOT)/bin/crane.exe:
	$(info ::: Downloading crane from $(CRANE_URL))
	@mkdir -p $(REPO_ROOT)/bin
	@curl -sSfL --retry 5 $(CRANE_URL) | tar zx -C $(REPO_ROOT)/bin crane.exe
else
# All other OSes (macos/Linux)
# Normalize architecture for go-containerregistry filenames
CRANE_ARCH = $(subst amd64,x86_64,$(BUILDARCH))
CRANE_OS = $(shell uname -s)

# Install crane binary into bin/
bin/crane: $(REPO_ROOT)/bin/crane
$(REPO_ROOT)/bin/crane:
	$(info ::: Downloading crane from $(CRANE_URL))
	@mkdir -p $(REPO_ROOT)/bin
	@tmp=$$(mktemp -d $(REPO_ROOT)/bin/.crane.XXXXXX) && trap 'rm -rf "$$tmp"' EXIT && \
		$(call fetch_file,$(CRANE_URL),"$$tmp/crane.tar.gz") && \
		tar xz -C "$$tmp" -f "$$tmp/crane.tar.gz" crane && \
		chmod +x "$$tmp/crane" && \
		mv "$$tmp/crane" "$@"
endif # Windows_NT

###############################################################################
# Common functions for launching a local Kubernetes control plane.
###############################################################################
# Shell test that passes only while the named container is up. A stopped container
# still answers docker inspect, so checking for existence alone reuses a dead one.
container_running = [ "$$(docker inspect -f '{{.State.Running}}' $(1) 2>/dev/null)" = "true" ]

## Kubernetes apiserver used for tests
APISERVER_NAME := calico-local-apiserver
run-k8s-apiserver: run-etcd
	@if $(call container_running,$(APISERVER_NAME)); then \
		echo "$(APISERVER_NAME) already running"; \
	else \
		docker rm -f $(APISERVER_NAME) >/dev/null 2>&1 || true; \
		docker run --detach --net=host \
			--name $(APISERVER_NAME) \
			-v $(REPO_ROOT):/go/src/github.com/projectcalico/calico \
			-v $(CERTS_PATH):/home/user/certs \
			-e KUBECONFIG=/home/user/certs/kubeconfig \
			$(CALICO_BUILD) kube-apiserver \
			--etcd-servers=http://$(LOCAL_IP_ENV):2379 \
			--service-cluster-ip-range=10.101.0.0/16,fd00:96::/112 \
			--authorization-mode=RBAC \
			--service-account-key-file=/home/user/certs/service-account.pem \
			--service-account-signing-key-file=/home/user/certs/service-account-key.pem \
			--service-account-issuer=https://localhost:443 \
			--api-audiences=kubernetes.default \
			--client-ca-file=/home/user/certs/ca.pem \
			--tls-cert-file=/home/user/certs/kubernetes.pem \
			--tls-private-key-file=/home/user/certs/kubernetes-key.pem \
			--enable-priority-and-fairness=false \
			--max-mutating-requests-inflight=0 \
			--max-requests-inflight=0 \
			--enable-aggregator-routing \
			--requestheader-client-ca-file=/home/user/certs/ca.pem \
			--requestheader-allowed-names=kubernetes \
			--requestheader-username-headers=X-Remote-User \
			--requestheader-group-headers=X-Remote-Group \
			--requestheader-extra-headers-prefix=X-Remote-Extra- \
			--proxy-client-cert-file=/home/user/certs/kubernetes.pem \
			--proxy-client-key-file=/home/user/certs/kubernetes-key.pem; \
		while ! docker exec $(APISERVER_NAME) kubectl get nodes; do echo "Waiting for apiserver to come up..."; sleep 2; done; \
		while ! docker exec $(APISERVER_NAME) kubectl get clusterrolebinding anonymous-admin >/dev/null 2>&1 && \
			! docker exec $(APISERVER_NAME) kubectl create \
			clusterrolebinding anonymous-admin \
			--clusterrole=cluster-admin \
			--user=system:anonymous 2>/dev/null; \
			do echo "Waiting for $(APISERVER_NAME) to come up"; \
			sleep 1; \
			done; \
		while ! docker exec $(APISERVER_NAME) kubectl \
			apply -f /go/src/github.com/projectcalico/calico/$(CALICO_CRD_PATH); \
			do echo "Trying to create CRDs"; \
			sleep 1; \
			done; \
	fi

# Stop Kubernetes apiserver
stop-k8s-apiserver:
	@-docker rm -f $(APISERVER_NAME)

# Run a local Kubernetes controller-manager in a docker container, useful for tests.
CONTROLLER_MANAGER_NAME := calico-local-controller-manager
run-k8s-controller-manager: run-k8s-apiserver
	@if $(call container_running,$(CONTROLLER_MANAGER_NAME)); then \
		echo "$(CONTROLLER_MANAGER_NAME) already running"; \
	else \
		docker rm -f $(CONTROLLER_MANAGER_NAME) >/dev/null 2>&1 || true; \
		docker run --detach --net=host \
			--name $(CONTROLLER_MANAGER_NAME) \
			-v $(CERTS_PATH):/home/user/certs \
			$(CALICO_BUILD) kube-controller-manager \
			--master=https://127.0.0.1:6443 \
			--kubeconfig=/home/user/certs/kube-controller-manager.kubeconfig \
			--min-resync-period=3m \
			--allocate-node-cidrs=true \
			--cluster-cidr=192.168.0.0/16 \
			--v=5 \
			--service-account-private-key-file=/home/user/certs/service-account-key.pem \
			--root-ca-file=/home/user/certs/ca.pem; \
	fi

## Stop Kubernetes controller manager
stop-k8s-controller-manager:
	@-docker rm -f $(CONTROLLER_MANAGER_NAME)

###############################################################################
# Common functions for create a local kind cluster.
###############################################################################
KIND_DIR := $(REPO_ROOT)/hack/test/kind

# An Enterprise install ships every operator CRD in the tree, so the kind rig
# creates the whole directory rather than the Calico subset.
OPERATOR_CRDS_SRC = $(REPO_ROOT)/operator/pkg/crds/operator
KIND ?= $(KIND_DIR)/kind
KUBECTL ?= $(KIND_DIR)/kubectl

# Different tests may require different kind configurations.
KIND_CONFIG ?= $(KIND_DIR)/kind.config
KIND_NAME = $(basename $(notdir $(KIND_CONFIG)))
KIND_KUBECONFIG?=$(KIND_DIR)/$(KIND_NAME)-kubeconfig.yaml

# Dataplane - kind supports iptables, nftables and bpf.
KIND_DATAPLANE ?= iptables
ifeq ($(filter $(KIND_DATAPLANE),iptables nftables bpf),)
  $(error KIND_DATAPLANE must be iptables, nftables or bpf, got "$(KIND_DATAPLANE)")
endif

# kube-proxy mode to render into the kind config. kube-proxy has no "bpf" mode
# (valid modes are iptables/ipvs/nftables); in a BPF-dataplane cluster kube-proxy
# runs in iptables mode at start-of-day and is then disabled by the operator
# (kubeProxyManagement). So map the BPF dataplane onto an iptables kube-proxy.
ifeq ($(KIND_DATAPLANE),bpf)
  KIND_KUBE_PROXY_MODE := iptables
else
  KIND_KUBE_PROXY_MODE := $(KIND_DATAPLANE)
endif

# Point the kind nodes at a DNS forwarder instead of the address Docker rewrites.
# Opt-in: it rewrites every node's /etc/resolv.conf. See dns-forwarder.sh.
KIND_DNS_FORWARDER?=false
KIND_DNS_FORWARDER_IMAGE?=coredns/coredns:$(COREDNS_VERSION)

# The external node runs docker-in-docker. Pinned so a new upstream tag cannot
# change the lane under us. See external-node.sh.
EXTERNAL_NODE_IMAGE?=docker:$(DOCKER_DIND_VERSION)-dind

.PHONY: kind-registry-up kind-registry-destroy
## Start the local kind image registry (idempotent). Persists across cluster recreates.
kind-registry-up:
	$(REPO_ROOT)/hack/test/kind/registry.sh up

## Remove the local kind image registry. Forces a full re-push on next kind-up.
# `make push` skips work based on `.dev-stamps/*pushed-id` (keyed on local
# image IDs), so dropping the registry alone would leave it empty after the
# next kind-build-images. Clear the push stamps so the next build re-pushes.
kind-registry-destroy:
	$(REPO_ROOT)/hack/test/kind/registry.sh down
	rm -f $(REPO_ROOT)/.dev-stamps/*pushed-id 2>/dev/null || true

kind-cluster-create: $(REPO_ROOT)/.$(KIND_NAME).created
$(REPO_ROOT)/.$(KIND_NAME).created: $(KUBECTL) $(KIND) kind-registry-up
	# First make sure any previous cluster is deleted
	$(MAKE) kind-cluster-destroy

	# Render the kind config with the requested kube-proxy mode.
	sed 's/^\(  mode: \)iptables$$/\1$(KIND_KUBE_PROXY_MODE)/' $(KIND_CONFIG) > $(KIND_DIR)/.$(KIND_NAME).rendered.config

	# calico-fluent-bit's in_tail (several tail inputs) plus per-node kube-proxy
	# exhaust the default inotify instance limit (128) across the kind nodes,
	# crash-looping the calico-fluent-bit pods with "too many open files" and
	# failing kind-deploy. Raise the limits before creating the cluster (same
	# values as hack/test/kind/vrf/vrf.sh). Non-interactive (sudo -n) so it never
	# blocks a headless run; best-effort warn (don't fail) if we can't set them,
	# e.g. no passwordless sudo on a dev box.
	sudo -n sysctl -w fs.inotify.max_user_instances=512 fs.inotify.max_user_watches=524288 || echo "WARNING: could not raise inotify limits; calico-fluent-bit may crash-loop with 'too many open files'"

ifeq ($(KIND_DATAPLANE),bpf)
	# The BPF lane is IPv4-only (see kind-bpf.config), to mirror the IPv4-only
	# GCP BPF job. kind creates its "kind" docker network with an IPv6 ULA by
	# default, which gives every node an IPv6 address; the node resolver then
	# returns AAAA records for external domains, and kind's IPv6 ULA has no
	# internet route, so host-networked egress probes (the DNS-policy tests) hang
	# on the unroutable IPv6 address. Pre-create the "kind" network IPv4-only so
	# kind reuses it instead of creating a dual-stack one. If a previous
	# dual-stack cluster left an IPv6-enabled network behind, recreate it
	# (disconnect its endpoints first, e.g. the persistent kind-registry, which
	# registry.sh reconnects below). Refuse if another kind cluster's nodes are
	# attached: the network is shared, so recreating it would break that cluster.
	@if docker network inspect kind --format '{{range .IPAM.Config}}{{.Subnet}} {{end}}' 2>/dev/null | grep -q ':'; then \
	  foreign=$$(for c in $$(docker network inspect kind --format '{{range .Containers}}{{.Name}} {{end}}' 2>/dev/null); do \
	      cl=$$(docker inspect "$$c" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}' 2>/dev/null); \
	      if [ -n "$$cl" ] && [ "$$cl" != "$(KIND_NAME)" ]; then echo "$$cl"; fi; \
	    done | sort -u | tr '\n' ' '); \
	  if [ -n "$$foreign" ]; then \
	    echo "ERROR: the 'kind' network is IPv6-enabled and must be recreated IPv4-only for the BPF lane,"; \
	    echo "       but nodes of another kind cluster are attached: $$foreign"; \
	    echo "       Recreating it would break that cluster. Delete it first:"; \
	    echo "         kind delete cluster --name <name>"; \
	    exit 1; \
	  fi; \
	  echo "Recreating the 'kind' docker network as IPv4-only for the BPF lane"; \
	  for c in $$(docker network inspect kind --format '{{range .Containers}}{{.Name}} {{end}}' 2>/dev/null); do docker network disconnect -f kind "$$c" 2>/dev/null || true; done; \
	  docker network rm kind 2>/dev/null || true; \
	fi
	@docker network inspect kind >/dev/null 2>&1 || docker network create kind
endif

	# Create a kind cluster.
	$(KIND) create cluster \
		--config $(KIND_DIR)/.$(KIND_NAME).rendered.config \
		--kubeconfig $(KIND_KUBECONFIG) \
		--name $(KIND_NAME) \
		--image kindest/node:$(KINDEST_NODE_VERSION)

	# Connect the registry to the kind network now that the network exists,
	# and write per-node containerd config so localhost:5000 redirects to it.
	$(REPO_ROOT)/hack/test/kind/registry.sh up
	KIND_NAME=$(KIND_NAME) $(REPO_ROOT)/hack/test/kind/registry.sh configure-nodes

	# After the registry, so the kind-registry resolution check is meaningful.
ifeq ($(KIND_DNS_FORWARDER),true)
	KIND_DNS_FORWARDER_IMAGE=$(KIND_DNS_FORWARDER_IMAGE) $(REPO_ROOT)/hack/test/kind/dns-forwarder.sh up
	KIND_NAME=$(KIND_NAME) $(REPO_ROOT)/hack/test/kind/dns-forwarder.sh configure-nodes
endif

	# Wait for controller manager to be running and healthy, then create Calico CRDs.
	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) get serviceaccount default; do echo "Waiting for default serviceaccount to be created..."; sleep 2; done
	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) create -f $(OPERATOR_CRDS_SRC)/; do echo "Waiting for operator CRDs to be created"; sleep 2; done
	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) create -f $(REPO_ROOT)/$(CALICO_CRD_PATH); do echo "Waiting for calico CRDs to be created"; sleep 2; done
	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) create -f $(REPO_ROOT)/charts/crd.projectcalico.org.v1/templates/eck/; do echo "Waiting for ECK CRDs to be created"; sleep 2; done

	# These may have already been created, depending on where we're getting our CRDs from. So use apply.
	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f $(REPO_ROOT)/libcalico-go/config/crd/policy.networking.k8s.io_adminnetworkpolicies.yaml; do echo "Waiting for ANP CRDs to be created"; sleep 2; done
	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f $(REPO_ROOT)/libcalico-go/config/crd/policy.networking.k8s.io_baselineadminnetworkpolicies.yaml; do echo "Waiting for CRDs to be created"; sleep 2; done

	# Install mutating admission policies (only for v3 CRDs, since they reference projectcalico.org/v3 resources).
ifeq ($(CALICO_API_GROUP),projectcalico.org/v3)
	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f $(REPO_ROOT)/api/admission/; do echo "Waiting for mutating admission policies to be created"; sleep 2; done
endif

	# Install the CNI annotation admission policies, which apply whatever the API group.
	while ! KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f $(REPO_ROOT)/api/admission/cni-annotations/; do echo "Waiting for admission policies to be created"; sleep 2; done

	touch $@

kind-cluster-destroy kind-down: $(KIND) $(KUBECTL)
	# Tear down the e2e external node (if any) alongside the cluster. Idempotent
	# and a no-op when no external node was created (e.g. non-BPF jobs).
	-$(KIND_DIR)/external-node.sh down
	# Unconditional: a lane that did not opt in must still clean up after one that did.
	KIND_NAME=$(KIND_NAME) $(KIND_DIR)/dns-forwarder.sh down || true
	# We need to drain the cluster gracefully when shutting down to avoid a netdev unregister error from the kernel.
	# This requires we execute CNI del on pods with pod networking.
	-$(KIND) delete cluster --name $(KIND_NAME)
	rm -f $(KIND_KUBECONFIG)
	rm -f $(KIND_DIR)/.$(KIND_NAME).rendered.config
	rm -f $(REPO_ROOT)/.$(KIND_NAME).created

$(KIND)-$(KIND_VERSION):
	mkdir -p $(KIND_DIR)/$(KIND_VERSION)
	$(DOCKER_GO_BUILD) sh -c "GOBIN=/go/src/github.com/projectcalico/calico/hack/test/kind/$(KIND_VERSION) go install sigs.k8s.io/kind@$(KIND_VERSION)"
	mv $(KIND_DIR)/$(KIND_VERSION)/kind $(KIND_DIR)/kind-$(KIND_VERSION)
	rm -r $(KIND_DIR)/$(KIND_VERSION)

$(KIND_DIR)/.kind-updated-$(KIND_VERSION): $(KIND)-$(KIND_VERSION)
	rm -f $(KIND_DIR)/.kind-updated-*
	cd $(KIND_DIR) && ln -fs kind-$(KIND_VERSION) kind
	touch $@

.PHONY: kind
kind: $(KIND)
	@echo "kind: $(KIND)"
$(KIND): $(KIND_DIR)/.kind-updated-$(KIND_VERSION)

$(KUBECTL)-$(K8S_VERSION):
	mkdir -p $(KIND_DIR)
	curl -fsSL --retry 5 https://dl.k8s.io/release/$(K8S_VERSION)/bin/linux/$(ARCH)/kubectl -o $@
	chmod +x $@

$(KIND_DIR)/.kubectl-updated-$(K8S_VERSION): $(KUBECTL)-$(K8S_VERSION)
	rm -f $(KIND_DIR)/.kubectl-updated-*
	cd $(KIND_DIR) && ln -fs kubectl-$(K8S_VERSION) kubectl
	touch $@

.PHONY: kubectl
kubectl: $(KUBECTL)
	@echo "kubectl: $(KUBECTL)"
$(KUBECTL): $(KIND_DIR)/.kubectl-updated-$(K8S_VERSION)

bin/helm-$(HELM_VERSION):
	mkdir -p bin
	curl -sSfL --retry 5 --retry-all-errors -o /tmp/calico-helm.tar.gz https://get.helm.sh/helm-$(HELM_VERSION)-linux-$(ARCH).tar.gz
	tar xz --strip-components 1 -C bin -f /tmp/calico-helm.tar.gz linux-$(ARCH)/helm
	rm -f /tmp/calico-helm.tar.gz
	mv bin/helm bin/helm-$(HELM_VERSION)

bin/.helm-updated-$(HELM_VERSION): bin/helm-$(HELM_VERSION)
	# Remove old marker files so that bin/helm will be stale if we switch
	# branches and the helm version changes.
	rm -f bin/.helm-updated-*
	cd bin && ln -fs helm-$(HELM_VERSION) helm
	touch $@

.PHONY: helm
helm: bin/helm
	@echo "helm: $^"
bin/helm: bin/.helm-updated-$(HELM_VERSION)

# helm-gcs 0.7.0 and up need Helm 4 (platformHooks); every branch still pins Helm 3.
helm-install-gcs-plugin:
	bin/helm plugin install https://github.com/hayorov/helm-gcs.git --version 0.6.3

publish-charts: publish-charts-gcs publish-charts-oci
# Upload to Google tigera-helm-charts storage bucket. --retry reloads index.yaml when a
# concurrent nightly writes it first, otherwise that branch's entries are silently lost.
# --force allows republishing a version whose SHA has not moved since the last run.
# The read-back is the real guard: a lost merge still exits 0, and a for loop reports only
# its last iteration, so without both this and the || exit 1 the job goes green regardless.
publish-charts-gcs:
	bin/helm repo add tigera gs://tigera-helm-charts
	for chart in ./bin/*.tgz; do \
		bin/helm gcs push $$chart tigera --force --retry || exit 1; \
	done
	bin/helm repo update tigera
	for chart in ./bin/*.tgz; do \
		bin/helm search repo "tigera/$$(bin/helm show chart $$chart | awk '/^name:/{print $$2}')" \
			--version "$$(bin/helm show chart $$chart | awk '/^version:/{print $$2}')" \
			--fail-on-no-result >/dev/null || exit 1; \
	done

CHART_REPO?=oci://us-central1-docker.pkg.dev/unique-caldron-775/charts
publish-charts-oci:
	# publish charts to the new enterprise oci repo.
	gcloud auth application-default print-access-token | helm registry login -u oauth2accesstoken --password-stdin https://us-central1-docker.pkg.dev
	find ./bin -maxdepth 1 -type f -name "*.tgz" -print0 | xargs -0 -I {} bin/helm push {} $(CHART_REPO)

bin/yq:
	mkdir -p bin
	$(call fetch_file,https://github.com/mikefarah/yq/releases/download/v4.53.3/yq_linux_$(BUILDARCH).tar.gz,/tmp/calico-yq.tar.gz)
	tar xz -C bin -f /tmp/calico-yq.tar.gz ./yq_linux_$(BUILDARCH)
	rm -f /tmp/calico-yq.tar.gz
	mv bin/yq_linux_$(BUILDARCH) bin/yq

###############################################################################
# Dev image build variables. Used by the root Makefile's image and push
# targets, and by the operator image marker below.
###############################################################################
DEV_IMAGE_PATH ?= calico
DEV_IMAGE_TAG ?= $(GIT_VERSION)
DEV_IMAGE_REGISTRY ?= gcr.io/unique-caldron-775/dev
DEV_STAMP_DIR := $(REPO_ROOT)/.dev-stamps

# Map tigera/<name>:test-build → $(DEV_IMAGE_REGISTRY)/$(DEV_IMAGE_PATH)/<name>:$(DEV_IMAGE_TAG)
# notdir strips the org prefix (tigera/ or calico/) to get just the image name.
DEV_IMAGE_PREFIX = $(if $(filter docker.io,$(DEV_IMAGE_REGISTRY)),$(DEV_IMAGE_PATH),$(DEV_IMAGE_REGISTRY)/$(DEV_IMAGE_PATH))
DEV_CALICO_IMAGES = $(foreach img,$(KIND_CALICO_IMAGES),$(DEV_IMAGE_PREFIX)/$(notdir $(firstword $(subst :, ,$(img)))):$(DEV_IMAGE_TAG))
DEV_ENTERPRISE_IMAGES = $(foreach img,$(KIND_ENTERPRISE_LOCAL_IMAGES) $(KIND_ENTERPRISE_PULLED_IMAGES),$(DEV_IMAGE_PREFIX)/$(notdir $(firstword $(subst :, ,$(img)))):$(DEV_IMAGE_TAG))
DEV_OPERATOR_IMAGE = $(DEV_IMAGE_PREFIX)/operator:$(DEV_IMAGE_TAG)

###############################################################################
# Common functions for setting up a kind cluster with Calico for testing.
###############################################################################
KIND_INFRA_DIR := $(REPO_ROOT)/hack/test/kind/infra
KIND_TEST_BUILD_TAG = test-build

# Locally-built Calico images. dev-build.sh re-tags each as
# localhost:5000/tigera/<name>:test-build for the kind cluster.
# Most components are provided by the combined tigera/calico image, which
# uses "calico <subcommand>" as the container command. Only node remains
# as a separate image (it ships its own runit/bird/iptables filesystem).
KIND_CALICO_IMAGES = \
	tigera/node:$(KIND_TEST_BUILD_TAG) \
	tigera/calico:$(KIND_TEST_BUILD_TAG) \
	tigera/envoy-gateway:$(KIND_TEST_BUILD_TAG) \
	tigera/envoy-proxy:$(KIND_TEST_BUILD_TAG) \
	tigera/envoy-ratelimit:$(KIND_TEST_BUILD_TAG) \
	tigera/third-party-cni-plugins:$(KIND_TEST_BUILD_TAG)

# Enterprise-only: paths to license and pull secret for kind cluster setup.
GCR_IO_PULL_SECRET ?= $(HOME)/secrets/docker_cfg.json
TSEE_TEST_LICENSE ?= $(HOME)/secrets/license.yaml

# Enterprise images built locally with latest-$(ARCH) tags. kind-build-images
# pushes each as localhost:5000/tigera/<name>:test-build for the kind cluster.
KIND_ENTERPRISE_LOCAL_IMAGES = \
	tigera/egress-gateway:$(KIND_TEST_BUILD_TAG) \
	tigera/fluent-bit:$(KIND_TEST_BUILD_TAG) \
	tigera/deep-packet-inspection:$(KIND_TEST_BUILD_TAG) \
	tigera/intrusion-detection-controller:$(KIND_TEST_BUILD_TAG) \
	tigera/intrusion-detection-job-installer:$(KIND_TEST_BUILD_TAG) \
	tigera/gateway-l7-collector:$(KIND_TEST_BUILD_TAG) \
	tigera/prometheus-operator:$(KIND_TEST_BUILD_TAG) \
	tigera/prometheus-config-reloader:$(KIND_TEST_BUILD_TAG) \
	tigera/eck-operator:$(KIND_TEST_BUILD_TAG) \
	tigera/istio-install-cni:$(KIND_TEST_BUILD_TAG) \
	tigera/istio-pilot:$(KIND_TEST_BUILD_TAG) \
	tigera/istio-proxyv2:$(KIND_TEST_BUILD_TAG) \
	tigera/istio-ztunnel:$(KIND_TEST_BUILD_TAG)

# Enterprise images pulled from a registry. Stamp rules pull and tag as
# latest-$(ARCH); the common tagging loop handles the final tag.
KIND_ENTERPRISE_PULLED_IMAGES = \
	tigera/prometheus:$(KIND_TEST_BUILD_TAG) \
	tigera/alertmanager:$(KIND_TEST_BUILD_TAG) \
	tigera/manager:$(KIND_TEST_BUILD_TAG) \
	tigera/elasticsearch:$(KIND_TEST_BUILD_TAG) \
	tigera/kibana:$(KIND_TEST_BUILD_TAG)

# All enterprise images = locally built + pulled from registry.
KIND_CALICO_ENTERPRISE_IMAGES = $(KIND_ENTERPRISE_LOCAL_IMAGES) $(KIND_ENTERPRISE_PULLED_IMAGES)

# .image.created markers: the per-component image build stamp files.
# Each depends on its source files via deps.txt so Make knows when
# to rebuild. The sub-make handles the actual build; we just ensure it
# runs when sources are newer.
KIND_IMAGE_MARKERS = \
	$(REPO_ROOT)/node/.image.created-$(ARCH) \
	$(REPO_ROOT)/cmd/calico/.image.created-$(ARCH) \
	$(REPO_ROOT)/operator/.image.created-$(ARCH) \
	$(REPO_ROOT)/third_party/envoy-gateway/.envoy-gateway.created-$(ARCH) \
	$(REPO_ROOT)/third_party/envoy-proxy/.envoy-proxy.created-$(ARCH) \
	$(REPO_ROOT)/third_party/envoy-ratelimit/.envoy-ratelimit.created-$(ARCH) \
	$(REPO_ROOT)/third_party/cni-plugins/.cni-plugins.created-$(ARCH)

# Shared libbpf marker. Both node and cmd/calico (and felix-internal
# build steps invoked from them) need libbpf, and `kind-build-images` runs
# the per-image markers under `make -j`. Without a shared prereq, the
# parallel sub-makes race in felix/bpf-gpl on both the clone (each runs
# `rm -rf libbpf && git clone`, one fails with "destination path 'libbpf'
# already exists") and the compile (both write to the same .o paths).
# Point both image markers at the compiled libbpf.a so Make builds it
# exactly once, serially, before the parallel image builds start.
LIBBPF_MARKER = $(REPO_ROOT)/felix/bpf-gpl/libbpf/src/$(ARCH)/libbpf.a

$(LIBBPF_MARKER):
	$(MAKE) -C $(REPO_ROOT)/felix libbpf ARCH=$(ARCH)

# MISSING-IMAGE forces a rebuild when the previously-built image has been
# pruned from the local Docker daemon. Each stamp records the image ref
# that was built; hack/image-exists prints MISSING-IMAGE (a phony,
# always-out-of-date target) when that ref is gone, dragging the rule
# back in. The stamp is removed before the sub-make so the sub-make's
# own up-to-date check doesn't short-circuit the rebuild.
.PHONY: MISSING-IMAGE
MISSING-IMAGE:

$(REPO_ROOT)/node/.image.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/node/.image.created-$(ARCH)) \
    $(LIBBPF_MARKER) $(call local-deps-go-files,node) $(call local-deps-go-files,cmd)
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/node image
	echo "node:latest-$(ARCH)" > $@

$(REPO_ROOT)/whisker/.image.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/whisker/.image.created-$(ARCH))
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/whisker image
	echo "whisker:latest-$(ARCH)" > $@

$(REPO_ROOT)/cmd/calico/.image.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/cmd/calico/.image.created-$(ARCH)) \
    $(LIBBPF_MARKER) $(call local-deps-go-files,cmd)
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/cmd/calico image
	echo "calico:latest-$(ARCH)" > $@

$(REPO_ROOT)/key-cert-provisioner/.image.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/key-cert-provisioner/.image.created-$(ARCH)) \
    $(call local-deps-go-files,key-cert-provisioner)
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/key-cert-provisioner image
	echo "test-signer:latest-$(ARCH)" > $@

# The operator bakes the component refs it installs into the image, so
# image-exists gets the expected ref: a changed DEV_IMAGE_* triple must
# rebuild even though the recorded image still exists.
$(REPO_ROOT)/operator/.image.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/operator/.image.created-$(ARCH) $(DEV_OPERATOR_IMAGE)) \
    $(call local-deps-go-files,operator)
	rm -f $@
	DEV_IMAGE_TAG=$(DEV_IMAGE_TAG) \
	  DEV_IMAGE_REGISTRY=$(DEV_IMAGE_REGISTRY) \
	  DEV_IMAGE_PATH=$(DEV_IMAGE_PATH) \
	  $(KIND_INFRA_DIR)/build-operator.sh
	echo "$(DEV_OPERATOR_IMAGE)" > $@

# Enterprise-only image markers for locally-built components. These follow
# the same pattern as KIND_IMAGE_MARKERS: source deps so Make knows when
# to rebuild. Components with deps.txt use local-deps-go-files; third_party
# components without deps.txt approximate with find.
KIND_ENTERPRISE_IMAGE_MARKERS = \
	$(REPO_ROOT)/egress-gateway/.image.created-$(ARCH) \
	$(REPO_ROOT)/fluent-bit/.fluent-bit.created-$(ARCH) \
	$(REPO_ROOT)/intrusion-detection-controller/.image.created-$(ARCH) \
	$(REPO_ROOT)/deep-packet-inspection/.image.created-$(ARCH) \
	$(REPO_ROOT)/gateway/.gateway-l7-collector.created-$(ARCH) \
	$(REPO_ROOT)/third_party/prometheus-operator/.prometheus-operator.created-$(ARCH) \
	$(REPO_ROOT)/third_party/prometheus-operator/.prometheus-config-reloader.created-$(ARCH) \
	$(REPO_ROOT)/third_party/eck-operator/.eck-operator.created-$(ARCH) \
	$(REPO_ROOT)/istio/.istio-cni.created-$(ARCH)

# Enterprise-only markers for images pulled from a registry. These have
# no local source deps and are pulled once per stamp lifetime.
KIND_ENTERPRISE_PULLED_IMAGE_MARKERS = \
	$(REPO_ROOT)/.stamp.prometheus \
	$(REPO_ROOT)/.stamp.alertmanager \
	$(REPO_ROOT)/.stamp.manager \
	$(REPO_ROOT)/.stamp.elastic \
	$(REPO_ROOT)/.stamp.kibana

# Envoy components: the third_party/envoy-* sub-makes use their own marker
# names (.envoy-<comp>.created-$(ARCH)). The sub-make handles fetching
# upstream sources and building the image as calico/envoy-<comp>:latest-$(ARCH).
$(REPO_ROOT)/third_party/envoy-gateway/.envoy-gateway.created-$(ARCH):
	$(MAKE) -C $(REPO_ROOT)/third_party/envoy-gateway image

$(REPO_ROOT)/third_party/envoy-proxy/.envoy-proxy.created-$(ARCH):
	$(MAKE) -C $(REPO_ROOT)/third_party/envoy-proxy image

$(REPO_ROOT)/third_party/envoy-ratelimit/.envoy-ratelimit.created-$(ARCH):
	$(MAKE) -C $(REPO_ROOT)/third_party/envoy-ratelimit image

# Istio integration images (istiod/pilot, install-cni, proxyv2 waypoint,
# ztunnel). The operator's versions files (calico_versions.yml /
# enterprise_versions.yml) already reference these at the KIND test-build tag,
# but the istio/ component Makefile is standalone and not otherwise wired into
# the kind build, so the operator's istio install fails to pull them
# (localhost:5000/tigera/istio-*:test-build) and the Istio waypoint e2e test
# fails. `make -C istio image` fetches the upstream sources and builds all four
# images (touching every .istio-*.created-$(ARCH) stamp), so a single marker on
# the install-cni stamp drives the whole set; the other three images are picked
# up by the tag/push loop via KIND_ENTERPRISE_LOCAL_IMAGES.
$(REPO_ROOT)/istio/.istio-cni.created-$(ARCH):
	$(MAKE) -C $(REPO_ROOT)/istio init-sources image ARCH=$(ARCH)

# third-party-cni-plugins clones the upstream CNI and flannel sources and
# compiles them, tagging the image as calico/third-party-cni-plugins:latest-$(ARCH).
$(REPO_ROOT)/third_party/cni-plugins/.cni-plugins.created-$(ARCH):
	$(MAKE) -C $(REPO_ROOT)/third_party/cni-plugins image

# The registry/path/tag every kind lane bakes into its images.
# hack/test/kind/infra/values.yaml pins the same triple, and the operator FV
# stamps it into the test binary.
KIND_IMAGE_REGISTRY = localhost:5000
KIND_IMAGE_PATH     = tigera
KIND_DEV_IMAGE_ARGS = \
	    DEV_IMAGE_REGISTRY=$(KIND_IMAGE_REGISTRY) \
	    DEV_IMAGE_PATH=$(KIND_IMAGE_PATH) \
	    DEV_IMAGE_TAG=$(KIND_TEST_BUILD_TAG)

## Build all component images and push them to the local kind registry.
# This invokes the same `make push` pipeline used by the release flow, with
# kind-flavored DEV_IMAGE_REGISTRY/PATH/TAG so images land at
# localhost:5000/tigera/<name>:test-build. dev-build.sh handles tag,
# operator-build (via build-operator.sh dev-mode), and push. The host pushes
# via the 127.0.0.1:5000 port mapping on the registry container; containerd
# inside the kind nodes mirrors localhost:5000 to http://kind-registry:5000.
.PHONY: kind-build-images
kind-build-images: kind-registry-up
	$(MAKE) -C $(REPO_ROOT) push $(KIND_DEV_IMAGE_ARGS)

## Build only the operator image, with the kind component references baked in.
# CI's "Build: operator image" block runs this and caches the result so the
# kind lanes load it instead of compiling the operator per job.
.PHONY: kind-operator-image
kind-operator-image:
	$(MAKE) -C $(REPO_ROOT) operator-image $(KIND_DEV_IMAGE_ARGS)

# Create a kind cluster and deploy Calico on it via Helm. Assumes images are
# already built and tagged as test-build in the local Docker daemon. If a
# cluster already exists (stamp file present), the creation step is skipped.
# Default to v3 CRDs for kind clusters. Override with KIND_CALICO_API_GROUP=crd.projectcalico.org/v1 if needed.
KIND_CALICO_API_GROUP ?= projectcalico.org/v3

# Install Calico via Helm and wait for readiness on an existing kind cluster.
# Images are pulled from the local kind-registry on demand, so no image
# loading happens here. Use kind-up for end-to-end bringup.
.PHONY: kind-deploy
kind-deploy:
	$(MAKE) -C $(REPO_ROOT) chart CALICO_API_GROUP=$(KIND_CALICO_API_GROUP)
	REPO_ROOT=$(REPO_ROOT) \
	KUBECONFIG=$(KIND_KUBECONFIG) \
	KIND=$(KIND) \
	KIND_NAME=$(KIND_NAME) \
	ARCH=$(ARCH) \
	GIT_VERSION=$(GIT_VERSION) \
	CALICO_API_GROUP=$(KIND_CALICO_API_GROUP) \
	KIND_DATAPLANE=$(KIND_DATAPLANE) \
	TSEE_TEST_LICENSE=$(TSEE_TEST_LICENSE) \
	GCR_IO_PULL_SECRET=$(GCR_IO_PULL_SECRET) \
	KIND_SKIP_TIGERASTATUS_WAIT=$(KIND_SKIP_TIGERASTATUS_WAIT) \
	$(REPO_ROOT)/hack/test/kind/deploy_resources.sh

# Rebuild any images whose source files have changed, push changed layers to
# the local registry, upgrade the Helm release, and restart pods so kubelet
# re-pulls the new digests under the test-build tag (PullAlways).
.PHONY: kind-reload
kind-reload:
	$(MAKE) -j$(NUM_BUILD_JOBS) kind-build-images
	$(MAKE) -C $(REPO_ROOT) chart CALICO_API_GROUP=$(KIND_CALICO_API_GROUP)
	KUBECONFIG=$(KIND_KUBECONFIG) $(REPO_ROOT)/bin/helm upgrade calico \
		$(REPO_ROOT)/bin/tigera-operator-$(GIT_VERSION).tgz \
		--reuse-values \
		-n tigera-operator
	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) delete pods -n calico-system --all
	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f $(KIND_INFRA_DIR)/calicoctl.yaml

# Enterprise-only image build rules.

# Locally-built enterprise components with deps.txt: these use the same
# .image.created pattern as OSS components so Make tracks source changes.
$(REPO_ROOT)/egress-gateway/.image.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/egress-gateway/.image.created-$(ARCH)) \
    $(call local-deps-go-files,egress-gateway)
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/egress-gateway image
	echo "egress-gateway:latest-$(ARCH)" > $@

# fluent-bit uses its own marker name (.fluent-bit.created-$(ARCH)) rather than
# .image.created-$(ARCH); its Makefile creates the marker, like gateway.
$(REPO_ROOT)/fluent-bit/.fluent-bit.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/fluent-bit/.fluent-bit.created-$(ARCH)) \
    $(call local-deps-go-files,fluent-bit)
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/fluent-bit image

# gateway uses its own marker name (.gateway-l7-collector.created-$(ARCH))
# rather than .image.created-$(ARCH); follow the same source-tracking pattern.
$(REPO_ROOT)/gateway/.gateway-l7-collector.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/gateway/.gateway-l7-collector.created-$(ARCH)) \
    $(call local-deps-go-files,gateway)
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/gateway image

# IDC's controller image is a thin wrapper over cmd/calico (adds GeoIP),
# so it must be sequenced after cmd/calico's image stamp. Without this,
# parallel `make -j` fires two concurrent cmd/calico builds (one from
# kind-build-images, one recursed from IDC's Makefile) that race on the
# shared docker buildx state.
#
# Grouped target (&:) — `make image` in this dir builds both the IDS
# controller and job-installer images. Without grouping, parallel `make
# -j` fires two sub-makes that race on the shared
# bin/intrusion-detection-job-installer binary and the job-installer
# docker image tag.
$(REPO_ROOT)/intrusion-detection-controller/.image.created-$(ARCH) $(REPO_ROOT)/intrusion-detection-controller/.dashboards-installer.image.created-$(ARCH) &: \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/intrusion-detection-controller/.image.created-$(ARCH)) \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/intrusion-detection-controller/.dashboards-installer.image.created-$(ARCH)) \
    $(REPO_ROOT)/cmd/calico/.image.created-$(ARCH) \
    $(call local-deps-go-files,intrusion-detection-controller)
	rm -f $(REPO_ROOT)/intrusion-detection-controller/.image.created-$(ARCH)
	rm -f $(REPO_ROOT)/intrusion-detection-controller/.dashboards-installer.image.created-$(ARCH)
	$(MAKE) -C $(REPO_ROOT)/intrusion-detection-controller image
	echo "intrusion-detection-controller:latest-$(ARCH)" > $(REPO_ROOT)/intrusion-detection-controller/.image.created-$(ARCH)
	echo "intrusion-detection-job-installer:latest-$(ARCH)" > $(REPO_ROOT)/intrusion-detection-controller/.dashboards-installer.image.created-$(ARCH)

$(REPO_ROOT)/deep-packet-inspection/.image.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/deep-packet-inspection/.image.created-$(ARCH)) \
    $(call local-deps-go-files,deep-packet-inspection)
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/deep-packet-inspection image
	echo "deep-packet-inspection:latest-$(ARCH)" > $@

# Third-party locally-built images: use find for source deps since these
# don't have deps.txt. The component Makefiles create the marker files.
# Grouped target (&:) — a single sub-make produces both stamps. Without
# grouping, parallel `make -j` fires two sub-makes that race on the shared
# prometheus-operator/ source tarball and corrupt the embedded YAML
# (unexpected length 583168 != 794599) during go build.
#
# Skip on Windows: these stamps are KIND/Linux-only, and the bare $(shell find ...)
# fires at parse time with `find` unavailable in PowerShell.
ifneq ($(OS),Windows_NT)
$(REPO_ROOT)/third_party/prometheus-operator/.prometheus-operator.created-$(ARCH) $(REPO_ROOT)/third_party/prometheus-operator/.prometheus-config-reloader.created-$(ARCH) &: \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/third_party/prometheus-operator/.prometheus-operator.created-$(ARCH)) \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/third_party/prometheus-operator/.prometheus-config-reloader.created-$(ARCH)) \
    $(shell find $(REPO_ROOT)/third_party/prometheus-operator -name '*.go')
	rm -f $(REPO_ROOT)/third_party/prometheus-operator/.prometheus-operator.created-$(ARCH)
	rm -f $(REPO_ROOT)/third_party/prometheus-operator/.prometheus-config-reloader.created-$(ARCH)
	$(MAKE) -C $(REPO_ROOT)/third_party/prometheus-operator image
	echo "prometheus-operator:latest-$(ARCH)" > $(REPO_ROOT)/third_party/prometheus-operator/.prometheus-operator.created-$(ARCH)
	echo "prometheus-config-reloader:latest-$(ARCH)" > $(REPO_ROOT)/third_party/prometheus-operator/.prometheus-config-reloader.created-$(ARCH)

$(REPO_ROOT)/third_party/eck-operator/.eck-operator.created-$(ARCH): \
    $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/third_party/eck-operator/.eck-operator.created-$(ARCH)) \
    $(shell find $(REPO_ROOT)/third_party/eck-operator -name '*.go')
	rm -f $@
	$(MAKE) -C $(REPO_ROOT)/third_party/eck-operator image
	echo "eck-operator:latest-$(ARCH)" > $@
endif

# GCR-pulled images: no local source deps, use stamp files. Tag as
# latest-$(ARCH) so the common tagging loop handles the final tag.
$(REPO_ROOT)/.stamp.prometheus: $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/.stamp.prometheus)
	# TODO: Build this. We cannot at the moment because it relies on the host OS npm / go toolchain.
	rm -f $@
	docker pull gcr.io/unique-caldron-775/cnx/tigera/prometheus:$(THIRD_PARTY_RELEASE_BRANCH)
	docker tag gcr.io/unique-caldron-775/cnx/tigera/prometheus:$(THIRD_PARTY_RELEASE_BRANCH) tigera/prometheus:latest-$(ARCH)
	echo "tigera/prometheus:latest-$(ARCH)" > $@

$(REPO_ROOT)/.stamp.alertmanager: $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/.stamp.alertmanager)
	# TODO: Build this.
	rm -f $@
	docker pull gcr.io/unique-caldron-775/cnx/tigera/alertmanager:$(THIRD_PARTY_RELEASE_BRANCH)
	docker tag gcr.io/unique-caldron-775/cnx/tigera/alertmanager:$(THIRD_PARTY_RELEASE_BRANCH) tigera/alertmanager:latest-$(ARCH)
	echo "tigera/alertmanager:latest-$(ARCH)" > $@

$(REPO_ROOT)/.stamp.manager: $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/.stamp.manager)
	# TODO
	rm -f $@
	docker pull gcr.io/unique-caldron-775/cnx/tigera/manager:$(THIRD_PARTY_RELEASE_BRANCH)
	docker tag gcr.io/unique-caldron-775/cnx/tigera/manager:$(THIRD_PARTY_RELEASE_BRANCH) tigera/manager:latest-$(ARCH)
	echo "tigera/manager:latest-$(ARCH)" > $@

$(REPO_ROOT)/.stamp.elastic: $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/.stamp.elastic)
	# TODO: Build this.
	rm -f $@
	docker pull gcr.io/unique-caldron-775/cnx/tigera/elasticsearch:$(THIRD_PARTY_RELEASE_BRANCH)
	docker tag gcr.io/unique-caldron-775/cnx/tigera/elasticsearch:$(THIRD_PARTY_RELEASE_BRANCH) tigera/elasticsearch:latest-$(ARCH)
	echo "tigera/elasticsearch:latest-$(ARCH)" > $@

$(REPO_ROOT)/.stamp.kibana: $(shell $(REPO_ROOT)/hack/image-exists $(REPO_ROOT)/.stamp.kibana)
	# Kibana is only deployed by the enterprise-suite lanes (e.g. the BPF e2e
	# lane enables it for the Feature:Kibana-Auth tests); pull it like the other
	# third-party base images rather than building it from source here.
	rm -f $@
	docker pull gcr.io/unique-caldron-775/cnx/tigera/kibana:$(THIRD_PARTY_RELEASE_BRANCH)
	docker tag gcr.io/unique-caldron-775/cnx/tigera/kibana:$(THIRD_PARTY_RELEASE_BRANCH) tigera/kibana:latest-$(ARCH)
	echo "tigera/kibana:latest-$(ARCH)" > $@

###############################################################################
# Common functions for setting up a local envtest environment.
###############################################################################
ENVTEST_DIR := $(REPO_ROOT)/hack/test/envtest
ENVTEST_CONTAINER_DIR := /go/src/github.com/projectcalico/calico/hack/test/envtest
# Derive major.minor from K8S_VERSION (e.g. v1.34.3 -> 1.34.x) for setup-envtest.
# Envtest publishes binaries per minor version, not per patch, so we use a wildcard.
# Skip on Windows: envtest is Linux-only test infra; bash sed/cut would error otherwise.
ifneq ($(OS),Windows_NT)
ENVTEST_K8S_VERSION ?= $(shell echo $(K8S_VERSION) | sed 's/^v//' | cut -d. -f1,2).x
endif
# Key the marker off the full K8S_VERSION rather than the minor wildcard. The
# fallback below stores its assets under k8s/<full version>-<os>-<arch>, so a
# bump inside one minor (rc to GA, say) has to force a fresh setup.
ENVTEST_ASSETS_MARKER := $(ENVTEST_DIR)/.envtest-$(K8S_VERSION:v%=%)

## Download envtest binaries (kube-apiserver, etcd, kubectl) for use by tests that use controller-runtime envtest.
.PHONY: setup-envtest
setup-envtest: $(ENVTEST_ASSETS_MARKER)
$(ENVTEST_ASSETS_MARKER):
	@echo "Setting up envtest binaries for Kubernetes $(ENVTEST_K8S_VERSION)..."
	mkdir -p $(ENVTEST_DIR)
	rm -f $(ENVTEST_DIR)/.envtest-*
	# Prefer the bundle controller-tools publishes. It only exists for minors it
	# has already cut an envtest release for, which trails the Kubernetes release
	# by anywhere from hours to a fortnight, and it never covers most
	# pre-releases. Assemble the same layout from the release binaries when it is
	# missing, so a Kubernetes bump is never blocked waiting on that release; the
	# published bundle takes over again on the next clean setup once it lands.
	# -alpha is excluded either way: too unstable to pin, so it fails loudly.
	$(DOCKER_GO_BUILD) sh -c 'set -e; \
		case "$(K8S_VERSION)" in \
			*-alpha*) echo "K8S_VERSION $(K8S_VERSION) is an alpha; envtest does not pin to alphas." >&2; exit 1;; \
		esac; \
		if go run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest \
			use --bin-dir $(ENVTEST_CONTAINER_DIR) -p path $(ENVTEST_K8S_VERSION); then \
			exit 0; \
		fi; \
		echo "No published envtest bundle for $(ENVTEST_K8S_VERSION); assembling one from the $(K8S_VERSION) release binaries."; \
		base=https://dl.k8s.io/release/$(K8S_VERSION)/bin/$(BUILDOS)/$(BUILDARCH); \
		d=$(ENVTEST_CONTAINER_DIR)/k8s/$(K8S_VERSION:v%=%)-$(BUILDOS)-$(BUILDARCH); \
		mkdir -p $$d; \
		for b in kube-apiserver kubectl; do \
			curl -fsSL --retry 5 -o $$d/$$b $$base/$$b; \
			curl -fsSL --retry 5 -o $$d/$$b.sha256 $$base/$$b.sha256; \
			echo "$$(cat $$d/$$b.sha256)  $$d/$$b" | sha256sum -c -; \
			rm -f $$d/$$b.sha256; \
		done; \
		etcd_tgz=etcd-$(ETCD_VERSION)-$(BUILDOS)-$(BUILDARCH).tar.gz; \
		etcd_url=https://github.com/etcd-io/etcd/releases/download/$(ETCD_VERSION); \
		curl -fsSL --retry 5 -o /tmp/$$etcd_tgz $$etcd_url/$$etcd_tgz; \
		curl -fsSL --retry 5 -o /tmp/etcd.SHA256SUMS $$etcd_url/SHA256SUMS; \
		echo "$$(grep -F "$$etcd_tgz" /tmp/etcd.SHA256SUMS | cut -d" " -f1)  /tmp/$$etcd_tgz" | sha256sum -c -; \
		tar -xzf /tmp/$$etcd_tgz -C /tmp; \
		mv /tmp/etcd-$(ETCD_VERSION)-$(BUILDOS)-$(BUILDARCH)/etcd $$d/etcd; \
		chmod +x $$d/kube-apiserver $$d/kubectl $$d/etcd'
	touch $@

# Minimum supported Kubernetes version. 1.32 is the first release that estimates
# the cost of the CEL IP/CIDR library well enough for our CRD rules to install.
MIN_K8S_VERSION ?= v1.32.0
ifneq ($(OS),Windows_NT)
ENVTEST_MIN_K8S_VERSION ?= $(shell echo $(MIN_K8S_VERSION) | sed 's/^v//' | cut -d. -f1,2).x
# Major.minor prefix for globbing the downloaded envtest directory (e.g. "1.29").
ENVTEST_MIN_K8S_MINOR := $(shell echo $(MIN_K8S_VERSION) | sed 's/^v//' | cut -d. -f1,2)
endif
ENVTEST_MIN_ASSETS_MARKER := $(ENVTEST_DIR)/.envtest-min-$(ENVTEST_MIN_K8S_VERSION)

## Download envtest binaries for the minimum supported Kubernetes version.
.PHONY: setup-envtest-min
setup-envtest-min: $(ENVTEST_MIN_ASSETS_MARKER)
$(ENVTEST_MIN_ASSETS_MARKER):
	@echo "Setting up envtest binaries for minimum K8s $(ENVTEST_MIN_K8S_VERSION)..."
	mkdir -p $(ENVTEST_DIR)
	$(DOCKER_GO_BUILD) sh -c \
		'go run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest \
		use --bin-dir $(ENVTEST_CONTAINER_DIR) -p path $(ENVTEST_MIN_K8S_VERSION)'
	touch $@

###############################################################################
# Common functions for launching a local etcd instance.
###############################################################################
## Run etcd as a container (calico-etcd)
# TODO: We shouldn't need to enable the v2 API, but some of our test code still relies on it.
.PHONY: run-etcd stop-etcd
run-etcd:
	@if ! $(call container_running,calico-etcd); then \
		docker rm -f calico-etcd >/dev/null 2>&1 || true; \
		docker run --detach \
			--net=host \
			--entrypoint=/usr/local/bin/etcd \
			--name calico-etcd $(ETCD_IMAGE) \
			--enable-v2 \
			--advertise-client-urls "http://$(LOCAL_IP_ENV):2379,http://127.0.0.1:2379,http://$(LOCAL_IP_ENV):4001,http://127.0.0.1:4001" \
			--listen-client-urls "http://0.0.0.0:2379,http://0.0.0.0:4001"; \
	fi

stop-etcd:
	@-docker rm -f calico-etcd

###############################################################################
# Helpers
###############################################################################
## Help
.PHONY: help
help:
	$(info Available targets)
	@echo
	@awk '/^[a-zA-Z\-\_\%0-9\/]+:/ {                                  \
	   nb = sub( /^## /, "", helpMsg );                               \
	   if(nb == 0) {                                                  \
	      helpMsg = $$0;                                              \
	      nb = sub( /^[^:]*:.* ## /, "", helpMsg );                   \
	   }                                                              \
	   if (nb)                                                        \
	      printf "\033[1;31m%-" width "s\033[0m %s\n", $$1, helpMsg;  \
	}                                                                 \
	{ helpMsg = $$0 }'                                                \
	width=30                                                          \
	$(MAKEFILE_LIST)
	@echo
	@echo "-----------------------------------------------------------"
	@echo "Building for $(BUILDOS)-$(ARCH) INSTALL_FLAG=$(INSTALL_FLAG)"
	@echo
	@echo "ARCH (target):		$(ARCH)"
	@echo "OS (target):		$(BUILDOS)"
	@echo "BUILDARCH (host):	$(BUILDARCH)"
	@echo "CALICO_BUILD:		$(CALICO_BUILD)"
	@echo "-----------------------------------------------------------"

###############################################################################
# Common functions for launching a local Elastic instance.
###############################################################################
ELASTIC_USERNAME := elastic
# Skip on Windows: bash-only pipeline (cat /dev/urandom, tr, head); the local-elastic
# targets below run on Linux only and Windows components don't reference these vars.
ifneq ($(OS),Windows_NT)
BOOTSTRAP_PASSWORD := $(shell cat /dev/urandom | LC_CTYPE=C tr -dc A-Za-z0-9 | head -c16)
ELASTIC_PASSWORD := $(BOOTSTRAP_PASSWORD)

# metadata.mk carries the version, but a Makefile that includes lib.Makefile without it
# still needs a tag here, so fall back to reading the file.
ELASTIC_IMAGE   ?= docker.elastic.co/elasticsearch/elasticsearch:$(if $(ELASTIC_VERSION),$(ELASTIC_VERSION),$(shell grep -o '^ELASTIC_VERSION=[0-9.]*' $(REPO_ROOT)/metadata.mk | cut -d "=" -f 2))
endif
ELASTIC_EXTRA_DOCKER_ARGS ?=
ELASTIC_MEMORY ?= 2GB

## Run elasticsearch as a container (tigera-elastic)
.PHONY: run-elastic
run-elastic: $(REPO_ROOT)/.elasticsearch.created
$(REPO_ROOT)/.elasticsearch.created:
	# Run ES on Docker.
	docker run --detach \
	-m $(ELASTIC_MEMORY) \
	--net=host \
	--name=tigera-elastic \
	-e "discovery.type=single-node" \
	-e "xpack.security.enabled=false" \
	$(ELASTIC_EXTRA_DOCKER_ARGS) \
	$(ELASTIC_IMAGE)

	# Wait until ES is accepting requests.
	@while ! docker exec tigera-elastic curl localhost:9200 2> /dev/null; do echo "Waiting for Elasticsearch to come up..."; sleep 2; done
	touch $@

	# Configure elastic to ignore high watermark errors, since this is just for tests.
	curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings -d '{"transient": {"cluster.routing.allocation.disk.threshold_enabled": false }}'
	curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}'

	echo "Elastic Credentials: ${ELASTIC_USERNAME} ${ELASTIC_PASSWORD}" >> $(REPO_ROOT)/.elasticsearch.created

## Stop elasticsearch with name tigera-elastic
.PHONY: stop-elastic
stop-elastic:
	-docker rm -f tigera-elastic
	rm -rf $(REPO_ROOT)/.elasticsearch.created

###############################################################################
# Common functions for building windows images.
###############################################################################

DOCKER_MANIFEST_CMD := docker manifest

ifdef CONFIRM
DOCKER_MANIFEST = $(DOCKER_MANIFEST_CMD)
else
DOCKER_MANIFEST = echo [DRY RUN] $(DOCKER_MANIFEST_CMD)
endif

# Named per component so two components' Windows builds do not remove each other's.
WINDOWS_BUILDER = calico-windows-builder-$(notdir $(CURDIR))

# Clean up the docker builder used to create Windows image tarballs.
.PHONY: clean-windows-builder
clean-windows-builder:
	-docker buildx rm $(WINDOWS_BUILDER)
	# Drain the shared builder earlier releases left selected on the host with --use.
	-docker buildx rm calico-windows-builder

# Set up the docker builder used to create Windows image tarballs.
.PHONY: setup-windows-builder
setup-windows-builder: clean-windows-builder
	docker buildx create --name=$(WINDOWS_BUILDER) --platform windows/amd64

# FIXME: Use WINDOWS_HPC_VERSION and image instead of nanoserver and WINDOWS_VERSIONS when containerd v1.6 is EOL'd
# .PHONY: image-windows release-windows
# NOTE: WINDOWS_IMAGE_REQS must be defined with the requirements to build the windows
# image. These must be added as reqs to 'image-windows' (originally defined in
# lib.Makefile) on the specific package Makefile otherwise they are not correctly
# recognized.
# # Build Windows image with tag and possibly push it to $DEV_REGISTRIES
# image-windows-with-tag: var-require-all-WINDOWS_IMAGE-WINDOWS_DIST-WINDOWS_IMAGE_REQS-IMAGETAG
# 	push="$${PUSH:-false}"; \
# 	for registry in $(DEV_REGISTRIES); do \
# 		echo Building and pushing Windows image to $${registry}; \
# 		image="$${registry}/$(WINDOWS_IMAGE):$(IMAGETAG)"; \
# 		docker buildx build \
# 			--platform windows/amd64 \
# 			--output=type=image,push=$${push} \
# 			-t $${image} \
# 			--pull \
# 			--no-cache \
# 			--build-arg GIT_VERSION=$(GIT_VERSION) \
# 			--build-arg WINDOWS_HPC_VERSION=$(WINDOWS_HPC_VERSION) \
# 			-f Dockerfile.windows .; \
# 	done ;

# image-windows: var-require-all-BRANCH_NAME
# 	$(MAKE) image-windows-with-tag PUSH=$(PUSH) IMAGETAG=$(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(BRANCH_NAME)
# 	$(MAKE) image-windows-with-tag PUSH=$(PUSH) IMAGETAG=$(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(GIT_VERSION)

# # Build and push Windows image
# release-windows: var-require-one-of-CONFIRM-DRYRUN release-prereqs clean-windows
# 	$(MAKE) image-windows PUSH=true

# Windows image pushing is different because we do not build docker images directly.
# Since the build machine is linux, we output the images to a tarball. (We can
# produce images but there will be no output because docker images
# built for Windows cannot be loaded on linux.)
#
# The resulting image tarball is then pushed to registries during cd/release.
# The image tarballs are located in WINDOWS_DIST and have files names
# with the format 'node-windows-v3.21.0-2-abcdef-20H2.tar'.
#
# In addition to pushing the individual images, we also create the manifest
# directly using 'docker manifest'. This is possible because Semaphore is using
# a recent enough docker CLI version (20.10.0)
#
# - Create the manifest with 'docker manifest create' using the list of all images.
# - For each windows version, 'docker manifest annotate' its image with "os.image: <windows_version>".
#   <windows_version> is the version string that looks like, e.g. 10.0.19041.1288.
#   Setting os.image in the manifest is required for Windows hosts to load the
#   correct image in manifest.
# - Finally we push the manifest, "purging" the local manifest.

$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-%.tar: windows-sub-image-$*

# NOTE: WINDOWS_IMAGE_REQS must be defined with the requirements to build the windows
# image. These must be added as reqs to 'image-windows' (originally defined in
# lib.Makefile) on the specific package Makefile otherwise they are not correctly
# recognized.

# WINDOWS_BUILD_ARGS lets a component Makefile pass extra --build-arg flags to
# its Dockerfile.windows build (e.g. fluent-bit passes FLUENT_BIT_VERSION).
windows-sub-image-%: var-require-all-GIT_VERSION-WINDOWS_IMAGE-WINDOWS_DIST-WINDOWS_IMAGE_REQS
	# ensure dir for windows image tars exits
	-mkdir -p $(WINDOWS_DIST)
	docker buildx build \
		--builder $(WINDOWS_BUILDER) \
		--platform windows/amd64 \
		--output=type=docker,dest=$(CURDIR)/$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-$*.tar \
		$(DOCKER_PULL) \
		-t $(WINDOWS_IMAGE):latest \
		--build-arg GIT_VERSION=$(GIT_VERSION) \
		--build-arg THIRD_PARTY_REGISTRY=$(THIRD_PARTY_REGISTRY) \
		--build-arg THIRD_PARTY_RELEASE_BRANCH=$(THIRD_PARTY_RELEASE_BRANCH) \
		--build-arg WINDOWS_VERSION=$* \
		$(WINDOWS_BUILD_ARGS) \
		-f Dockerfile.windows .

.PHONY: image-windows release-windows release-windows-with-tag retag-windows-image-with-registries
image-windows: setup-windows-builder var-require-all-WINDOWS_VERSIONS
	for version in $(WINDOWS_VERSIONS); do \
		$(MAKE) windows-sub-image-$${version}; \
	done;

release-windows-with-tag: var-require-one-of-CONFIRM-DRYRUN var-require-all-IMAGETAG-DEV_REGISTRIES image-windows bin/crane
	for registry in $(DEV_REGISTRIES); do \
		echo Pushing Windows images to $${registry}; \
		all_images=""; \
		manifest_image="$${registry}/$(WINDOWS_IMAGE):$(IMAGETAG)"; \
		for win_ver in $(WINDOWS_VERSIONS); do \
			image_tar="$(WINDOWS_DIST)/$(WINDOWS_IMAGE)-$(GIT_VERSION)-$${win_ver}.tar"; \
			image="$${registry}/$(WINDOWS_IMAGE):$(IMAGETAG)-windows-$${win_ver}"; \
			echo Pushing image $${image} ...; \
			$(CRANE) push $${image_tar} $${image} & \
			all_images="$${all_images} $${image}"; \
		done; \
		wait; \
		$(DOCKER_MANIFEST) create --amend $${manifest_image} $${all_images}; \
		for win_ver in $(WINDOWS_VERSIONS); do \
			version=$$(docker manifest inspect mcr.microsoft.com/windows/nanoserver:$${win_ver} | jq -r '.manifests[0].platform."os.version"'); \
			image="$${registry}/$(WINDOWS_IMAGE):$(IMAGETAG)-windows-$${win_ver}"; \
			$(DOCKER_MANIFEST) annotate --os windows --arch amd64 --os-version $${version} $${manifest_image} $${image}; \
		done; \
		$(DOCKER_MANIFEST) push --purge $${manifest_image}; \
		$(RELEASE_PY3) $(QUAY_SET_EXPIRY_SCRIPT) add --expiry-days=$(QUAY_EXPIRE_DAYS) $${manifest_image} $${all_images} || true; \
	done;

# retag-windows-image-with-registries copies the Windows image from DEV_TAG to
# IMAGETAG in each registry. Windows images are single-arch manifests built by
# buildx, so they have no local per-arch images to retag.
retag-windows-image-with-registries: var-require-one-of-CONFIRM-DRYRUN var-require-all-DEV_REGISTRIES-WINDOWS_IMAGE-DEV_TAG-IMAGETAG bin/crane
	for registry in $(DEV_REGISTRIES); do \
		$(CRANE) cp $${registry}/$(WINDOWS_IMAGE):$(DEV_TAG) $${registry}/$(WINDOWS_IMAGE):$(IMAGETAG); \
	done;

release-windows: var-require-one-of-CONFIRM-DRYRUN var-require-all-DEV_REGISTRIES-WINDOWS_IMAGE var-require-one-of-VERSION-BRANCH_NAME bin/crane
	describe_tag=$(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(GIT_VERSION); \
	release_tag=$(if $(VERSION),$(VERSION),$(if $(IMAGETAG_PREFIX),$(IMAGETAG_PREFIX)-)$(BRANCH_NAME)); \
	$(MAKE) release-windows-with-tag IMAGETAG=$${describe_tag}; \
	for registry in $(DEV_REGISTRIES); do \
		$(CRANE) cp $${registry}/$(WINDOWS_IMAGE):$${describe_tag} $${registry}/$(WINDOWS_IMAGE):$${release_tag}; \
	done;

# Name of a test image that is used by both "node" and "calicoctl", so defined here.  This image is
# built by node/Makefile.
TEST_CONTAINER_NAME_VER?=latest
TEST_CONTAINER_NAME?=calico/test:$(TEST_CONTAINER_NAME_VER)-$(ARCH)