open-vault/packages-oss.lock/layer.mk

# WARNING: Do not EDIT or MERGE this file, it is generated by 'packagespec lock'.
# layer.mk contains the machinery to incrementally build the builder image
# as separate layers, so each can be cached both locally and in CI. This serves
# both to speed up builds by avoiding unnecessary repetition of work already done,
# as well as to ehnance the reliability of builds by downloading external
# dependencies only once per build when necessary.
#
# The build layers themselves can be individually exported as tarballs (by calling
# make <layer-name>-save) for later inspection, for sharing, or for implementing
# on-host caching without recourse to external docker registries.
#
# To use this file, include it in another makefile, and from there you must eval
# calls to the LAYER macro with this syntax:
#
#   $(eval $(call LAYER,<name>,<type>,<parent-name>,<source-include>,<source-exclude>))
#
# Each layer assumes the existence of a Dockerfile named <name>.Dockerfile in
# packages.lock/layers.
# It uses the <parent-name> to set a Docker build arg called BASE_IMAGE to the
# resultant docker image ref of the named parent layer. You should use this BASE_IMAGE
# in the FROM line in your image.
#
# There must also be a base image which has no parent, and that Dockerfile should
# use a FROM line from an explicit docker image, e.g. debian:buster.
#
# Each image is provided only the source code identified by <source-include>, minus
# any source code matched by <source-exclude>. Source code is any files which are
# present and not ignored by Git. This includes cached files, modified files and new,
# untracked files. The Dockerfile belonging to this layer is ALWAYS included in the
# source, so you don't need to manually specify that.
#
# The set of source code identified by a single image layer is used to produce its
# SOURCE_ID. The SOURCE_ID, when all the files are tracked by Git and are not modified
# equals the latest Git commit SHA that affected any of those files or directories.
# When there are any new or modified files, we take a SHA 256 sum of the latest Git
# commit affecting those files concatenated with the output of git diff and the contents
# of any untracked files, and prefix this with "dirty_". The SOURCE_ID is used as part of
# the cache key for that layer.
#
# Because different combinations of source-include and source-exclude may have been
# modified by the same commit, they may share the same source ID. Therefore, we also
# calculate the LAYER_ID which takes into account not only the current layer's source
# ID, but also its source include/exclude and the ID of its base layer. Thus any change
# in any of the inputs of any base layer invalidates the cache of all subsequent layers.

include $(shell git rev-parse --show-toplevel)/packages*.lock/config.mk

.SECONDARY:

DOCKERFILES_DIR := $(LOCKDIR)/layers

_ := $(shell mkdir -p $(CACHE_ROOT)/source-archives)

### END BUILDER IMAGE LAYERS

## LAYER

# The LAYER macro defines all the targets for each image defined above.
#
# The phony targets are the ones we typically run ourselves or in CI, they are:
#
#   <name>-debug     : dump debug info for this image layer
#   <name>-image     : build the image for this image layer
#   <name>-save      : save the docker image for this layer as a tar.gz
#   <name>-load      : load this image from the saved tar.gz

define LAYER
LAYERS += $(1)
$(1)_NAME           := $(1)
$(1)_TYPE           := $(2)
$(1)_BASE           := $(3)
$(1)_SOURCE_INCLUDE := $(4)
$(1)_SOURCE_EXCLUDE := $(sort $(5) $(ALWAYS_EXCLUDE_SOURCE))
$(1)_CACHE_KEY_FILE := $(REPO_ROOT)/$(6)
$(1)_IMAGE_ARCHIVE  := $(REPO_ROOT)/$(7)

$(1)_CACHE_ROOT     := $(CACHE_ROOT)/layers/$$($(1)_NAME)

ifneq ($$($(1)_BASE),)
$(1)_BASE_CACHE_ROOT := $(CACHE_ROOT)/layers/$$($(1)_BASE)
$(1)_BASE_ID_FILE    := $$($(1)_BASE_CACHE_ROOT)/current-layer-id
$(1)_BASE_LAYER_ID   := $$(shell cat $$($(1)_BASE_ID_FILE))
$(1)_BASE_CACHE      := $$($(1)_BASE_CACHE_ROOT)/$$($(1)_BASE_LAYER_ID)
$(1)_BASE_IMAGE      := $$($(1)_BASE_CACHE)/image.marker
$(1)_BASE_IMAGE_NAME  = $$(shell cat $$($(1)_BASE_IMAGE))
endif

$(1)_DOCKERFILE := $(DOCKERFILES_DIR)/$$($(1)_NAME).Dockerfile

# If no source is included, set source ID to none.
# Note that we include the checksum of the generated Dockerfile as part of cache IDs
# so we still invalidate the cache appropriately.
ifeq ($$($(1)_SOURCE_INCLUDE),)

$(1)_SOURCE_CMD          := echo ""
$(1)_SOURCE_ID           := packagespec-only-$$($(1)_NAME)
$(1)_SOURCE_ID_NICE_NAME := <packagespec-only>

else

$(1)_SOURCE_GIT = $$($(1)_SOURCE_INCLUDE) $$(call GIT_EXCLUDE_LIST,$$($(1)_SOURCE_EXCLUDE))
$(1)_SOURCE_COMMIT       := $$(shell git rev-list -n1 $(GIT_REF) -- $$($(1)_SOURCE_GIT))

# If we allow dirty builds, generate the source ID as a function of the
# source in in the current work tree. Where the source all happens to match a Git commit,
# that commit's SHA will be the source ID.
ifeq ($(ALLOW_DIRTY),YES)

$(1)_SOURCE_CMD := { { \
					  git ls-files -- $$($(1)_SOURCE_GIT); \
			 		  git ls-files -m --exclude-standard -- $$($(1)_SOURCE_GIT); \
			 	  } | sort | uniq; }
$(1)_SOURCE_MODIFIED     := $$(trim $$(shell git ls-files -m -- $$($(1)_SOURCE_GIT)))
$(1)_SOURCE_MODIFIED_SUM := $$(trim $$(shell git diff -- $$($(1)_SOURCE_GIT) | $(SUM)))
$(1)_SOURCE_NEW          := $$(trim $$(shell git ls-files -o --exclude-standard -- $$($(1)_SOURCE_GIT)))
$(1)_SOURCE_NEW_SUM      := $$(trim $$(shell git ls-files -o --exclude-standard -- $$($(1)_SOURCE_GIT) | $(SUM)))
$(1)_SOURCE_DIRTY        := $$(trim $$(shell if [ -z "$$($(1)_SOURCE_MODIFIED)" ] && [ -z "$$($(1)_SOURCE_NEW)" ]; then echo NO; else echo YES; fi))

$(1)_SOURCE_ID           := $$(shell if [ -z "$$($(1)_SOURCE_MODIFIED)" ] && [ -z "$$($(1)_SOURCE_NEW)" ]; then \
								   echo $$($(1)_SOURCE_COMMIT); \
				      		   else \
								   echo -n dirty_; echo $$($(1)_SOURCE_MODIFIED_SUM) $$($(1)_SOURCE_NEW_SUM) | $(SUM); \
							   fi)
$(1)_SOURCE_DIRTY_LIST   := $$($(1)_SOURCE_MODIFIED) $$($(1)_SOOURCE_NEW)

$(1)_SOURCE_ID_NICE_NAME := $$($(1)_SOURCE_ID)

# No dirty builds allowed, so the SOURCE_ID is the git commit SHA,
# and we list files using git ls-tree.
else

$(1)_SOURCE_ID           := $$($(1)_SOURCE_COMMIT)
$(1)_SOURCE_ID_NICE_NAME := $$($(1)_SOURCE_ID)
$(1)_SOURCE_CMD          := git ls-tree -r --name-only $(GIT_REF) -- $$($(1)_SOURCE_GIT)

endif
endif

# LAYER_ID_CONTENTS dictates all the fields that can cause cache invalidation
# to propagate from the current layer to all dependent layers.
define $(1)_LAYER_ID_CONTENTS
BASE_LAYER_ID=$$($(1)_BASE_LAYER_ID);
LAYER_NAME=$$($(1)_NAME);
SOURCE_ID=$$($(1)_SOURCE_ID);
SOURCE_INCLUDE=$$($(1)_SOURCE_INCLUDE);
SOURCE_EXCLUDE=$$($(1)_SOURCE_EXCLUDE);
endef

$(1)_LAYER_ID_CONTENTS_FILE := $$($(1)_CACHE_ROOT)/current-layer-id-contents
$(1)_LAYER_ID_FILE          := $$($(1)_CACHE_ROOT)/current-layer-id

# Create cache root dir and write LAYER_ID_FILE_CONTENTS file.
_ := $$(shell \
	mkdir -p $$($(1)_CACHE_ROOT); \
	echo "$$($(1)_LAYER_ID_CONTENTS)" > $$($(1)_LAYER_ID_CONTENTS_FILE); \
)

$(1)_LAYER_ID       := $$(shell cat $$($(1)_LAYER_ID_CONTENTS_FILE) | $(SUM))
$(1)_SOURCE_ARCHIVE := $(CACHE_ROOT)/source-archives/$$($(1)_TYPE)-$$($(1)_LAYER_ID).tar
$(1)_IMAGE_NAME     := $(BUILDER_IMAGE_PREFIX)-$$($(1)_NAME):$$($(1)_LAYER_ID)
$(1)_CACHE          := $(CACHE_ROOT)/layers/$$($(1)_NAME)/$$($(1)_LAYER_ID)

# Create cache dir and write Layer ID file.
_ := $$(shell \
	mkdir -p $$($(1)_CACHE); \
	echo $$($(1)_LAYER_ID) > $$($(1)_LAYER_ID_FILE); \
)

$(1)_PHONY_TARGET_NAMES := debug id image save load

$(1)_PHONY_TARGETS := $$(addprefix $$($(1)_NAME)-,$$($(1)_PHONY_TARGET_NAMES))

.PHONY: $$($(1)_PHONY_TARGETS)

# File targets.
$(1)_IMAGE             := $$($(1)_CACHE)/image.marker
$(1)_LAYER_REFS        := $$($(1)_CACHE)/image.layer_refs
$(1)_IMAGE_TIMESTAMP   := $$($(1)_CACHE)/image.created_time

$(1)_TARGETS = $$($(1)_PHONY_TARGETS)

# UPDATE_MARKER_FILE ensures the image marker file has the same timestamp as the
# docker image creation date it represents. This enables make to only rebuild it when
# it has really changed, especially after loading the image from an archive.
# It also writes a list of all the layers in this docker image's history, for use
# when saving layers out to archives for use in pre-populating Docker build caches.
define $(1)_UPDATE_MARKER_FILE
	export MARKER=$$($(1)_IMAGE); \
	export LAYER_REFS=$$($(1)_LAYER_REFS); \
	export IMAGE=$$($(1)_IMAGE_NAME); \
	export IMAGE_CREATED; \
	if ! { IMAGE_CREATED="$$$$(docker inspect -f '{{.Created}}' $$$$IMAGE 2>/dev/null)"; }; then \
		if [ -f "$$$$MARKER" ]; then \
			echo "==> Removing stale marker file for $$$$IMAGE" 1>&2; \
			rm -f $$$$MARKER; \
		fi; \
		exit 0; \
	fi; \
	if [ ! -f "$$$$MARKER" ]; then \
		echo "==> Writing marker file for $$$$IMAGE (created $$$$IMAGE_CREATED)" 1>&2; \
	fi; \
	echo $$$$IMAGE > $$$$MARKER; \
	$(TOUCH) -m -d $$$$IMAGE_CREATED $$$$MARKER; \
	echo "$$$$IMAGE" > $$$$LAYER_REFS; \
	docker history --no-trunc -q $$$$IMAGE | grep -Fv '<missing>' >> $$$$LAYER_REFS; 
endef

## PHONY targets
$(1)-debug:
	@echo "==> Debug info: $$($(1)_NAME) depends on $$($(1)_BASE)"
	@echo "$(1)_TARGETS               = $$($(1)_TARGETS)"
	@echo "$(1)_SOURCE_CMD            = $$($(1)_SOURCE_CMD)"
	@echo "$(1)_CACHE                 = $$($(1)_CACHE)"
	@echo "$(1)_DOCKERFILE            = $$($(1)_DOCKERFILE)"
	@echo "$(1)_SOURCE_COMMIT         = $$($(1)_SOURCE_COMMIT)"
	@echo "$(1)_SOURCE_ID             = $$($(1)_SOURCE_ID)"
	@echo "$(1)_SOURCE_MODIFIED       = $$($(1)_SOURCE_MODIFIED)"
	@echo "$(1)_SOURCE_DIRTY          = $$($(1)_SOURCE_DIRTY)"
	@echo "$(1)_SOURCE_NEW            = $$($(1)_SOURCE_NEW)"
	@echo "$(1)_IMAGE                 = $$($(1)_IMAGE)"
	@echo "$(1)_IMAGE_TIMESTAMP       = $$($(1)_IMAGE_TIMESTAMP)"
	@echo "$(1)_IMAGE_ARCHIVE         = $$($(1)_IMAGE_ARCHIVE)"
	@echo "$(1)_BASE_IMAGE            = $$($(1)_BASE_IMAGE)"
	@echo

$(1)-id:
	@echo $(1)-$$($(1)_SOURCE_ID)

$(1)-write-cache-key:
	@mkdir -p $$(dir $$($(1)_CACHE_KEY_FILE)); \
	cp $$($(1)_LAYER_ID_CONTENTS_FILE) $$($(1)_CACHE_KEY_FILE); \
	echo "==> Cache key for $(1) written to $$($(1)_CACHE_KEY_FILE)"; \
	cat $$($(1)_CACHE_KEY_FILE)

$(1)-image: $$($(1)_IMAGE)
	@cat $$<

$(1)-layer-refs: $$($(1)_LAYER_REFS)
	@echo $$<

$(1)-save: $$($(1)_IMAGE_ARCHIVE)
	@echo $$<

$(1)-load:
	@\
		ARCHIVE=$$($(1)_IMAGE_ARCHIVE); \
		IMAGE=$$($(1)_IMAGE_NAME); \
		MARKER=$$($(1)_IMAGE); \
		rm -f $$$$MARKER; \
		echo "==> Loading $$$$IMAGE image from $$$$ARCHIVE"; \
		docker load < $$$$ARCHIVE
	@$$(call $(1)_UPDATE_MARKER_FILE)

## END PHONY targets

# Set the BASE_IMAGE build arg to reference the appropriate base image,
# unless there is no referenced base image.
$(1)_DOCKER_BUILD_ARGS = $$(shell [ -z "$$($(1)_BASE)" ] || echo --build-arg BASE_IMAGE=$$$$(cat $$($(1)_BASE_IMAGE)))

$(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE := $$($(1)_CACHE)/source-archive.tar

# Build the docker image.
#
# For dirty builds, tar up a source archive from the local filesystem.
# We --ignore-failed-read so that deleted files that are not
# committed do not cause problems. This should be OK for dirty builds.
#
# For non-dirty builds, ask Git directly for a source archive.
#
# We explicitly set the TAR format to ustar because this seems more compatible
# with Docker than any other format. In future we should change this to POSIX
# once Docker supports that properly, because ustar only supports filenames
# < 256 chars which could eventually be an issue.
TAR_FORMAT := --format=ustar
$(1)_FULL_DOCKER_BUILD_COMMAND = docker build -t $$($(1)_IMAGE_NAME) $$($(1)_DOCKER_BUILD_ARGS) \
	-f $$($(1)_DOCKERFILE) - < $$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE)
$$($(1)_IMAGE): $$($(1)_BASE_IMAGE)
	@$$(call $(1)_UPDATE_MARKER_FILE)
	@if [ -f "$$@" ]; then exit 0; fi; \
	echo "==> Building Docker image $$($(1)_IMAGE_NAME)"; \
	echo "    Layer name             : $$($(1)_NAME)"; \
	echo "    Layer source ID        : $$($(1)_SOURCE_ID_NICE_NAME)"; \
	echo "    For product revision   : $(PRODUCT_REVISION_NICE_NAME)"; \
	echo "    For package source ID  : $(PACKAGE_SOURCE_ID)"; \
	if [ ! -f "$$($(1)_SOURCE_ARCHIVE)" ]; then \
		if [ "$(ALLOW_DIRTY)" = "YES" ]; then \
			echo "==> Building source archive from working directory: $$($(1)_SOURCE_ARCHIVE)" 1>&2; \
			$$($(1)_SOURCE_CMD) | $(TAR) --create $(TAR_FORMAT) --file $$($(1)_SOURCE_ARCHIVE) --ignore-failed-read -T -; \
		else \
			echo "==> Building source archive from git: $$($(1)_SOURCE_ARCHIVE)" 1>&2; \
			git archive --format=tar $(GIT_REF) $$($(1)_SOURCE_GIT) > $$($(1)_SOURCE_ARCHIVE); \
		fi; \
	fi; \
	if [ ! -f "$$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE)" ]; then \
		echo "==> Appending Dockerfile to source archive: $$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE)" 1>&2; \
		cp $$($(1)_SOURCE_ARCHIVE) $$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE); \
		$(TAR) --append $(TAR_FORMAT) $$($(1)_DOCKERFILE) --file $$($(1)_SOURCE_ARCHIVE_WITH_DOCKERFILE); \
	fi; \
	echo $$($(1)_FULL_DOCKER_BUILD_COMMAND); \
	$$($(1)_FULL_DOCKER_BUILD_COMMAND); \
	$$(call $(1)_UPDATE_MARKER_FILE)

# Save the docker image as a tar.gz.
$$($(1)_IMAGE_ARCHIVE): | $$($(1)_IMAGE)
	@mkdir -p $$(dir $$@); \
	IMAGE=$$$$(cat $$($(1)_IMAGE)); \
		echo "==> Saving $(1) image to $$@"; \
		docker save $$$$IMAGE \
			$$$$(docker history -q --no-trunc $$$$IMAGE | grep -v missing) \
			| gzip > $$@

$$($(1)_LAYER_REFS):
	@echo "$$($(1)_IMAGE_NAME)" > $$@
	@docker history --no-trunc -q $$($(1)_IMAGE_NAME) | grep -Fv '<missing>' >> $$@

endef

### END LAYER

# Include the generated instructions to build each layer.
include $(sort $(shell find $(DOCKERFILES_DIR) -name '*.mk'))

# Eagerly update the docker image marker files.
_ := $(foreach L,$(LAYERS),$(shell $(call $(L)_UPDATE_MARKER_FILE)))

# DOCKER_LAYER_LIST is used to dump the name of every docker ref in use
# by all of the current builder images. By running 'docker save' against
# this list, we end up with a tarball that can pre-populate the docker
# cache to avoid unnecessary rebuilds.
DOCKER_LAYER_LIST := $(CACHE_ROOT)/docker-layer-list

write-cache-keys: $(addsuffix -write-cache-key,$(LAYERS))
	@echo "==> All cache keys written."

build-all-layers: $(addsuffix -image,$(LAYERS))
	@echo "==> All builder layers built."

.PHONY: debug
debug: $(addsuffix -debug,$(LAYERS))