From 3987e82eec1bd08624a05637ac079b4dbce48799 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 21 Oct 2026 09:00:29 -0400 Subject: [PATCH] Build basic test images with rules_img Assemble Alpine, BusyBox and the tmpfile fixture as Bazel image targets so workers can build and cache them without a Docker daemon. Pin both architecture variants of the existing base images and declare the fixture file as a layer input. Connect the resulting archives to the existing Make image targets. Hash the assembled archive before looking up cached images so changes to transitive Bazel inputs invalidate the image tag. Preserve the image names consumed by tests and the existing registry push/load interface. Assisted-by: Codex --- MODULE.bazel | 32 ++++++++++++++++ images/README.md | 15 +++++++- images/basic/BUILD | 55 ++++++++++++++++++++++++++++ images/basic/alpine/Dockerfile | 1 - images/basic/busybox/Dockerfile | 1 - images/basic/tmpfile/Dockerfile | 4 -- images/basic/tmpfile/file.txt | 1 + images/img.bzl | 36 ++++++++++++++++++ tools/images.mk | 50 ++++++++++++++++++++++--- tools/licensecheck/dependencies.yaml | 24 +++++++++--- 10 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 images/basic/BUILD delete mode 100644 images/basic/alpine/Dockerfile delete mode 100644 images/basic/busybox/Dockerfile delete mode 100644 images/basic/tmpfile/Dockerfile create mode 100644 images/basic/tmpfile/file.txt create mode 100644 images/img.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 989c240aa45..d6e69541080 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -10,6 +10,7 @@ bazel_dep(name = "bazel_skylib", version = "1.9.2") bazel_dep(name = "bazel_lib", version = "3.2.2") bazel_dep(name = "yq.bzl", version = "0.4.0") bazel_dep(name = "rules_pkg", version = "1.0.1") +bazel_dep(name = "rules_img", version = "0.3.23") bazel_dep(name = "rules_cc", version = "0.2.25") bazel_dep(name = "llvm", version = "0.8.24") bazel_dep(name = "libbpf", version = "1.7.0") @@ -246,6 +247,37 @@ http_file( ], ) +### Docker integration test images +# Pin both architectures of the existing test distributions. Image assembly +# uses declared layers and does not require a Docker daemon. +img_pull = use_repo_rule("@rules_img//img:pull.bzl", "pull") + +img_pull( + name = "test_alpine", + digest = "sha256:b276d875eeed9c7d3f1cfa7edb06b22ed22b14219a7d67c52c56612330348239", + layer_handling = "eager", + platforms = [ + "linux/amd64", + "linux/arm64", + ], + registry = "index.docker.io", + repository = "library/alpine", + tag = "3.11.5", +) + +img_pull( + name = "test_busybox", + digest = "sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209", + layer_handling = "eager", + platforms = [ + "linux/amd64", + "linux/arm64", + ], + registry = "index.docker.io", + repository = "library/busybox", + tag = "1.31.1", +) + ### Toolchains register_toolchains("//:cc_toolchain_k8") diff --git a/images/README.md b/images/README.md index 83e681d47f3..1658f13b485 100644 --- a/images/README.md +++ b/images/README.md @@ -33,7 +33,20 @@ single images via individual `load` targets, or pull all images via a single ## Adding new images -To add a new image, create a new directory under `images` containing a +Images can be assembled with `rules_img`; see `basic/BUILD` for examples. +Group related images in one Bazel package. Declare immutable base images in +`MODULE.bazel` and expose each image with `image_outputs` from +`//images:img.bzl`. Its `image_name` preserves the image's test-facing name +independently of the Bazel package. Add the image's Make name and context +target to `BAZEL_IMAGE_CONTEXTS` in `tools/images.mk`. Make hashes the +assembled archive, including its transitive inputs, before checking the image +cache. The image can also be built directly with Bazel, without a Docker daemon: + +```sh +bazel build --config=x86_64 //images/basic:tmpfile_archive +``` + +For a Docker build, create a directory under `images` containing a Dockerfile and any other files that the image requires. You may choose to add to an existing subdirectory if applicable, or create a new one. diff --git a/images/basic/BUILD b/images/basic/BUILD new file mode 100644 index 00000000000..6aa216b792e --- /dev/null +++ b/images/basic/BUILD @@ -0,0 +1,55 @@ +load("@rules_img//img:image.bzl", "image_manifest") +load("@rules_img//img:layer.bzl", "file_metadata", "image_layer") +load("//images:img.bzl", "image_outputs") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +image_manifest( + name = "alpine", + base = "@test_alpine", + tags = ["manual"], +) + +image_outputs( + name = "alpine", + image = ":alpine", + image_name = "basic/alpine", +) + +image_manifest( + name = "busybox", + base = "@test_busybox", + tags = ["manual"], +) + +image_outputs( + name = "busybox", + image = ":busybox", + image_name = "basic/busybox", +) + +# The integration test checks that mounting /tmp preserves image contents. +image_layer( + name = "tmpfile_layer", + srcs = {"/tmp/foo/file.txt": "tmpfile/file.txt"}, + create_parent_directories = "enabled", + default_metadata = file_metadata(mode = "0644"), + # Keep the base image's sticky /tmp when adding its child directory. + file_metadata = {"/tmp": file_metadata(mode = "1777")}, +) + +image_manifest( + name = "tmpfile", + base = "@test_alpine", + layers = [":tmpfile_layer"], + tags = ["manual"], +) + +image_outputs( + name = "tmpfile", + image = ":tmpfile", + image_name = "basic/tmpfile", +) diff --git a/images/basic/alpine/Dockerfile b/images/basic/alpine/Dockerfile deleted file mode 100644 index 12b26040a91..00000000000 --- a/images/basic/alpine/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM alpine:3.11.5 diff --git a/images/basic/busybox/Dockerfile b/images/basic/busybox/Dockerfile deleted file mode 100644 index 79b3f683abe..00000000000 --- a/images/basic/busybox/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM busybox:1.31.1 diff --git a/images/basic/tmpfile/Dockerfile b/images/basic/tmpfile/Dockerfile deleted file mode 100644 index e3816c8cbfb..00000000000 --- a/images/basic/tmpfile/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -# Create file under /tmp to ensure files inside '/tmp' are not overridden. -FROM alpine:3.11.5 -RUN mkdir -p /tmp/foo \ - && echo 123 > /tmp/foo/file.txt diff --git a/images/basic/tmpfile/file.txt b/images/basic/tmpfile/file.txt new file mode 100644 index 00000000000..190a18037c6 --- /dev/null +++ b/images/basic/tmpfile/file.txt @@ -0,0 +1 @@ +123 diff --git a/images/img.bzl b/images/img.bzl new file mode 100644 index 00000000000..4784903c4f1 --- /dev/null +++ b/images/img.bzl @@ -0,0 +1,36 @@ +"""Test image archives and contexts for the Make image targets.""" + +load("@rules_img//img:load.bzl", "image_load") +load("@rules_pkg//pkg:mappings.bzl", "pkg_files") +load("//tools:defs.bzl", "pkg_tar") + +def image_outputs(name, image, image_name): + """Exposes a Docker-save archive and Make's content-addressed context. + + Args: + name: Prefix for the archive and context target names. + image: The image_manifest target. + image_name: Stable image name relative to images/, such as basic/alpine. + """ + image_load( + name = name + "_load", + image = image, + tag = "gvisor.dev/images/%s:latest" % image_name, + tags = ["manual"], + ) + native.filegroup( + name = name + "_archive", + srcs = [":" + name + "_load"], + output_group = "tarball", + tags = ["manual"], + ) + pkg_files( + name = name + "_files", + srcs = [":" + name + "_archive"], + renames = {":" + name + "_archive": "image.tar"}, + ) + pkg_tar( + name = name + "_context", + srcs = [":" + name + "_files"], + package_dir = "images/" + image_name, + ) diff --git a/tools/images.mk b/tools/images.mk index b81614cc3c4..c7bb921b6ee 100644 --- a/tools/images.mk +++ b/tools/images.mk @@ -39,14 +39,23 @@ endif DOCKER_BUILD_ARGS ?= +# Prepared image contexts invoke this file directly after Bazel has built them. +SHELL = /bin/bash +header ?= echo --- $(1) >&2 + # Note that the image prefixes used here must match the image mangling in # runsc/testutil.MangleImage. Names are mangled in this way to ensure that all # tests are using locally-defined images (that are consistent and idempotent). REMOTE_IMAGE_PREFIX ?= us-central1-docker.pkg.dev/gvisor-presubmit/gvisor-presubmit-images LOCAL_IMAGE_PREFIX ?= gvisor.dev/images -ALL_IMAGES := $(subst /,_,$(subst images/,,$(shell find images/ -name Dockerfile -o -name Dockerfile.$(ARCH) | xargs -n 1 dirname | sort -u))) +BAZEL_IMAGE_CONTEXTS := \ + basic_alpine=//images/basic:alpine_context \ + basic_busybox=//images/basic:busybox_context \ + basic_tmpfile=//images/basic:tmpfile_context +BAZEL_IMAGES := $(foreach entry,$(BAZEL_IMAGE_CONTEXTS),$(word 1,$(subst =, ,$(entry)))) +ALL_IMAGES := $(BAZEL_IMAGES) $(subst /,_,$(subst images/,,$(shell find images/ -name Dockerfile -o -name Dockerfile.$(ARCH) | sed 's,/[^/]*$$,,' | sort -u))) NON_TEST_IMAGES := gpu/ollama/bench\|gpu/vllm\|gpu/triton\|tpu/vllm -TEST_IMAGES := $(subst /,_,$(subst images/,,$(shell find images/ -name Dockerfile -o -name Dockerfile.$(ARCH) | xargs -n 1 dirname | sort -u | grep -v "$(NON_TEST_IMAGES)"))) +TEST_IMAGES := $(BAZEL_IMAGES) $(subst /,_,$(subst images/,,$(shell find images/ -name Dockerfile -o -name Dockerfile.$(ARCH) | sed 's,/[^/]*$$,,' | sort -u | grep -v "$(NON_TEST_IMAGES)"))) GPU_TEST_IMAGES := $(filter gpu_% tpu_%,$(TEST_IMAGES)) CPU_TEST_IMAGES := $(filter-out gpu_% tpu_%,$(TEST_IMAGES)) @@ -123,6 +132,7 @@ push-all-test-images: $(patsubst %,push-%,$(TEST_IMAGES)) # (depending on what's available for the given architecture). path = images/$(subst _,/,$(1)) dockerfile = $$(if [ -f "$(call path,$(1))/Dockerfile.$(ARCH)" ]; then echo Dockerfile.$(ARCH); else echo Dockerfile; fi) +image_inputs = $(wildcard $(call path,$(1))/image.tar $(call path,$(1))/Dockerfile $(call path,$(1))/Dockerfile.$(ARCH)) # The tag construct is used to memoize the image generated (see README.md). # This scheme is used to enable aggressive caching in a central repository, but @@ -140,7 +150,7 @@ latest_tag = \ docker tag $(call local_image,$(1)):$(call tag,$(1)) $(call local_image,$(1)):latest >&2 tag_exists = \ docker image inspect $(call local_image,$(1)):$(call tag,$(1)) &>/dev/null -cached_image = $(if $(wildcard $(call path,$(1))/Dockerfile $(call path,$(1))/Dockerfile.$(ARCH)),$(shell \ +cached_image = $(if $(call image_inputs,$(1)),$(shell \ docker image inspect $(call remote_image,$(1)):$(call tag,$(1)) >/dev/null 2>&1 && echo cached)) tag-%: ## Tag a local image. @$(call header,TAG $*) @@ -164,13 +174,19 @@ pull-%: register-cross ## Force a repull of the image. # conflicting with the bazel "build" terminology, which is used elsewhere. rebuild = \ $(call header,REBUILD $(1)) && \ + (if [ -f "$(call path,$(1))/image.tar" ]; then \ + docker load --input "$(call path,$(1))/image.tar" >&2 && \ + docker tag "gvisor.dev/images/$(subst _,/,$(1)):latest" "$(call remote_image,$(1)):$(call tag,$(1))" >&2 && \ + docker tag "$(call remote_image,$(1)):$(call tag,$(1))" "$(call remote_image,$(1)):latest" >&2; \ + else \ (T=$$(mktemp -d) && cp -a $(call path,$(1))/* $$T && \ set -x && docker build $(DOCKER_PLATFORM_ARGS) $(DOCKER_BUILD_ARGS) \ -f "$$T/$(call dockerfile,$(1))" \ -t "$(call remote_image,$(1)):$(call tag,$(1))" \ -t "$(call remote_image,$(1))":latest \ $$T >&2 && \ - rm -rf $$T) && \ + rm -rf $$T); \ + fi) && \ $(call local_tag,$(1)) && \ $(call latest_tag,$(1)) rebuild-%: register-cross ## Force rebuild an image locally. @@ -185,7 +201,7 @@ rebuild-%: register-cross ## Force rebuild an image locally. .SECONDEXPANSION: load-%: $$(if $$(call cached_image,$$*),tag-$$*,register-cross) ## Pull or build an image locally. @if [ "$<" = register-cross ]; then \ - if [ ! -f "$(call path,$*)/$(call dockerfile,$*)" ]; then \ + if [ -z "$(call image_inputs,$*)" ]; then \ echo "Image $* is not available on $$(uname -m), ignoring it." >&2; \ elif [ "$(SKIP_IMAGE_LOAD)" == true ]; then \ if ! $(call tag_exists,$*); then \ @@ -231,3 +247,27 @@ endif else @ endif + +# Hash and cache the built archive, including its transitive inputs, rather +# than only the BUILD file. The recursive Make operates on the prepared +# context and never invokes Bazel again. +ifneq ($(_IMAGE_CONTEXT_PREPARED),1) +_image_makefile := $(abspath $(lastword $(MAKEFILE_LIST))) +_image_root_makefile := $(abspath $(dir $(_image_makefile))/../Makefile) +_image_context_options = $(if $(filter x86_64 aarch64,$(ARCH)),--config=$(ARCH),$(error Unsupported image architecture $(ARCH))) +_image_context_operations := tag pull rebuild load test local-image push +_image_context_targets := $(foreach op,$(_image_context_operations),$(addprefix $(op)-,$(BAZEL_IMAGES))) +define native_image_context = +$(foreach op,$(_image_context_operations),$(op)-$(1)): _image_context_target := $(2) +endef +$(foreach entry,$(BAZEL_IMAGE_CONTEXTS),$(eval $(call native_image_context,$(word 1,$(subst =, ,$(entry))),$(word 2,$(subst =, ,$(entry)))))) +$(_image_context_targets): + @set -eu; image_context_dir=$$(mktemp -d); trap 'rm -rf "$$image_context_dir"' EXIT; \ + $(MAKE) --no-print-directory -C "$(dir $(_image_root_makefile))" -f "$(_image_root_makefile)" copy \ + TARGETS="$(_image_context_target)" DESTINATION="$$image_context_dir/contexts.tar" \ + OPTIONS="$(OPTIONS) $(_image_context_options)" && \ + tar --extract --file "$$image_context_dir/contexts.tar" --directory "$$image_context_dir" --same-permissions --no-same-owner && \ + cd "$$image_context_dir" && \ + $(MAKE) --no-print-directory -f "$(_image_makefile)" _IMAGE_CONTEXT_PREPARED=1 ARCH="$(ARCH)" "$@" +.PHONY: $(_image_context_targets) +endif diff --git a/tools/licensecheck/dependencies.yaml b/tools/licensecheck/dependencies.yaml index 2b35a3baf5f..b1d2048172f 100644 --- a/tools/licensecheck/dependencies.yaml +++ b/tools/licensecheck/dependencies.yaml @@ -642,10 +642,12 @@ license: Apache-2.0 - dependency: github.com/moby/sys/symlink version: v0.3.0 - retrieved: "2026-08-27" + retrieved: "2026-10-21" commit: cafbe42351600ca9b363e220722f66d96f6e71f4 sha256: c0036c97c9b7b7a246468165a968fedd974691161b083f1e7e99bafc58e57451 - license: Apache-2.0 + license: + - Apache-2.0 + - BSD-3-Clause - dependency: github.com/moby/sys/user version: v0.4.0 retrieved: "2026-08-27" @@ -1024,10 +1026,12 @@ license: BSD-3-Clause - dependency: gopkg.in/yaml.v2 version: v2.4.0 - retrieved: "2026-08-27" + retrieved: "2026-10-21" commit: "" sha256: ede49e27c4cca6cdd2ec719aed8ea4d363710cceb3d411e7a786fbdec0d391fd - license: Apache-2.0 + license: + - Apache-2.0 + - MIT - dependency: gopkg.in/yaml.v3 version: v3.0.1 retrieved: "2026-08-27" @@ -1047,10 +1051,12 @@ - MPL-2.0 - dependency: honnef.co/go/tools version: v0.5.1 - retrieved: "2026-08-27" + retrieved: "2026-10-21" commit: 56172d41b117cc2c2f99f65fe0a790c8d7d7ea66 sha256: d728ff392fc5b6f676a30c36e9d0a5b85f6f2e06b4ebbb121c27d965cbdffa11 - license: MIT + license: + - BSD-3-Clause + - MIT - dependency: k8s.io/api version: v0.32.3 retrieved: "2026-08-27" @@ -1163,6 +1169,12 @@ commit: 9792f1c079a7602347e0fd966542b6d6b35a6c64 sha256: c3e253237109ab2e2a8d3cb075688b98a6e6fce43d849849648e8e3a84f20d6f license: Apache-2.0 +- dependency: rules_img + version: https://github.com/bazel-contrib/rules_img/releases/download/v0.3.23/rules_img-v0.3.23.tar.gz + retrieved: "2026-10-21" + commit: 5a6bfd045c78fec2ad600e22ec697e65a7c49c27 + sha256: 22092bd8f72d915995ab75c803a12f8506b8cf56bc59bc540f7df7812cc49a1d + license: Apache-2.0 - dependency: rules_license version: https://github.com/bazelbuild/rules_license/releases/download/1.0.0/rules_license-1.0.0.tar.gz retrieved: "2026-08-27"