From df3c09e34cdfbc37638e85678b1aea157ad7d2e9 Mon Sep 17 00:00:00 2001 From: Andrei Gherzan Date: Sun, 18 Oct 2026 20:01:07 +0100 Subject: [PATCH] ci: reuse docker images across jobs We were rebuilding Docker images from scratch with --no-cache, tagged by PR number. With this, we were building multiple images per PR for the same context. This change tags the images by the git tree hash of the build context and only build if the tag is missing. This naturally covers rebuilding on build context changes. Finally, this change handles refreshing images every 7 days (for example to refresh OS packages) and prune by CI label after 14 days. The gap ensures an image a running job just picked up isn't pruned under it. Signed-off-by: Andrei Gherzan --- .github/actions/docker-build/action.yml | 45 ++++++++++++------- .github/actions/docker-clean-image/action.yml | 19 -------- .../actions/docker-prune-images/action.yml | 18 ++++++++ .github/workflows/compliance.yml | 12 +++-- .github/workflows/yocto-builds.yml | 12 +++-- .github/workflows/yocto-layer.yml | 12 +++-- 6 files changed, 63 insertions(+), 55 deletions(-) delete mode 100644 .github/actions/docker-clean-image/action.yml create mode 100644 .github/actions/docker-prune-images/action.yml diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 3ecab6a..e844206 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -8,31 +8,46 @@ inputs: docker_image: required: true description: "The name of the docker image" - id: - required: true - description: "Namespace for the image" + +outputs: + image: + description: "Name and tag of the docker image to run" + value: ${{ steps.build.outputs.image }} runs: using: "composite" steps: - name: Build the ${{ inputs.docker_image }} docker image + id: build shell: bash - # We run this unconditionally even if the change doesn't touch the - # relevant docker files because there is a chance that another PR (or - # something else) rebuilt the local image. For example if the first - # version of the PR included change for the relevant docker image but a - # subsequent push to the PR branch dropped them. In this way we rebuild - # the image to avoid using the changes from the previous push. + # The image is tagged with the git tree hash of the build context, so a + # tag always maps to the same Dockerfile and scripts. This lets jobs and + # PRs on the same host share the image. run: | + # Images older than max_age_days are rebuilt so that base image and + # package updates are picked up. + # Keep docker-prune-images at twice this. + max_age_days=7 + tag="$(git rev-parse --short=12 HEAD:.github/workflows/docker-images)" + image="${{ inputs.docker_image }}:$tag" + echo "image=$image" >> "$GITHUB_OUTPUT" + + if created="$(docker image inspect -f '{{.Created}}' "$image" 2>/dev/null)"; then + age_days=$(( ($(date +%s) - $(date -d "$created" +%s)) / 86400 )) + if [ "$age_days" -lt "$max_age_days" ]; then + echo "Reusing $image (built $age_days days ago)." + exit 0 + fi + echo "$image is $age_days days old, rebuilding..." + fi + cd .github/workflows/docker-images/ - # We build a temporary image namespaced by the PR number so we can - # handle multiple runners on the same host using the same docker - # storage. tries=3 n=1 until [ "$n" -gt "$tries" ]; do - echo "Building the docker image ${{ inputs.docker_image }}-${{ inputs.id }}... try $n..." - if docker build --no-cache . -f "${{ inputs.docker_image }}/Dockerfile" -t "${{ inputs.docker_image }}-${{ inputs.id }}"; then + echo "Building the docker image $image... try $n..." + if docker build --pull --no-cache . -f "${{ inputs.docker_image }}/Dockerfile" \ + --label meta-raspberrypi-ci=1 -t "$image"; then # This can fail if a dangling images cleaning job runs in # parallel. So we try this a couple of times to minimize # conflict. This is because while building, docker creates a @@ -44,4 +59,4 @@ runs: n=$((n+1)) done [ "$n" -le "$tries" ] - echo "Temporary image built in ${{ inputs.docker_image }}." + echo "Image $image built." diff --git a/.github/actions/docker-clean-image/action.yml b/.github/actions/docker-clean-image/action.yml deleted file mode 100644 index dfc18d9..0000000 --- a/.github/actions/docker-clean-image/action.yml +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-FileCopyrightText: Andrei Gherzan -# -# SPDX-License-Identifier: MIT - -name: "Cleanup docker storage" - -inputs: - docker_image: - required: true - description: "The name of the docker image" - -runs: - using: "composite" - steps: - - name: Cleanup temporary image - shell: bash - run: | - echo "Cleanup ${{ inputs.docker_image }} image..." - docker rmi "${{ inputs.docker_image }}" || true diff --git a/.github/actions/docker-prune-images/action.yml b/.github/actions/docker-prune-images/action.yml new file mode 100644 index 0000000..d10e385 --- /dev/null +++ b/.github/actions/docker-prune-images/action.yml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Andrei Gherzan +# +# SPDX-License-Identifier: MIT + +name: "Prune old CI docker images" + +runs: + using: "composite" + steps: + - name: Prune old CI docker images + shell: bash + # Only images built by docker-build carry the label. Keep this at twice + # its max_age_days so an image a running job relies on is never pruned. + run: | + echo "Pruning CI images older than 14 days..." + docker image prune -af \ + --filter label=meta-raspberrypi-ci=1 \ + --filter until=336h diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index a508b0d..c87df1c 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -17,22 +17,20 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Build a temporary DCO image + - name: Build the DCO image + id: docker uses: ./.github/actions/docker-build with: docker_image: dco-check - id: ${{ github.event.number }} - name: Do DCO check run: | docker run --rm --security-opt apparmor=unconfined \ --security-opt seccomp=unconfined \ -v "$GITHUB_WORKSPACE:/work:ro" \ --env "BASE_REF=$GITHUB_BASE_REF" \ - "dco-check-${{ github.event.number }}" - - name: Cleanup temporary docker image - uses: ./.github/actions/docker-clean-image - with: - docker_image: dco-check-${{ github.event.number }} + "${{ steps.docker.outputs.image }}" + - name: Prune old CI docker images + uses: ./.github/actions/docker-prune-images if: always() - name: Cleanup dangling docker images uses: ./.github/actions/docker-clean-dangling diff --git a/.github/workflows/yocto-builds.yml b/.github/workflows/yocto-builds.yml index dbff9f5..0bb390d 100644 --- a/.github/workflows/yocto-builds.yml +++ b/.github/workflows/yocto-builds.yml @@ -39,11 +39,11 @@ jobs: - name: Check for Yocto relevant changes id: changed-files-specific uses: ./.github/actions/changed-files - - name: Build a temporary yocto-builder image + - name: Build the yocto-builder image + id: docker uses: ./.github/actions/docker-build with: docker_image: yocto-builder - id: ${{ github.event.number }} if: steps.changed-files-specific.outputs.any_changed == 'true' - name: Build the image run: | @@ -59,13 +59,11 @@ jobs: --env "DL_DIR=$DL_DIR" \ --env "SSTATE_DIR=$SSTATE_DIR" \ --ulimit "nofile=1024:1048576" \ - "yocto-builder-${{ github.event.number }}" \ + "${{ steps.docker.outputs.image }}" \ /entrypoint-build.sh if: steps.changed-files-specific.outputs.any_changed == 'true' - - name: Cleanup temporary docker image - uses: ./.github/actions/docker-clean-image - with: - docker_image: yocto-builder-${{ github.event.number }} + - name: Prune old CI docker images + uses: ./.github/actions/docker-prune-images if: always() - name: Cleanup dangling docker images uses: ./.github/actions/docker-clean-dangling diff --git a/.github/workflows/yocto-layer.yml b/.github/workflows/yocto-layer.yml index 673aeea..1b43b41 100644 --- a/.github/workflows/yocto-layer.yml +++ b/.github/workflows/yocto-layer.yml @@ -22,24 +22,22 @@ jobs: - name: Check for Yocto relevant changes id: changed-files-specific uses: ./.github/actions/changed-files - - name: Build a temporary yocto-builder image + - name: Build the yocto-builder image + id: docker uses: ./.github/actions/docker-build with: docker_image: yocto-builder - id: ${{ github.event.number }} if: steps.changed-files-specific.outputs.any_changed == 'true' - name: Run yocto-check-layer run: | docker run --rm --security-opt apparmor=unconfined \ -v "$GITHUB_WORKSPACE:/work:ro" \ --env "BASE_REF=$GITHUB_BASE_REF" \ - "yocto-builder-${{ github.event.number }}" \ + "${{ steps.docker.outputs.image }}" \ /entrypoint-yocto-check-layer.sh if: steps.changed-files-specific.outputs.any_changed == 'true' - - name: Cleanup temporary docker image - uses: ./.github/actions/docker-clean-image - with: - docker_image: yocto-builder-${{ github.event.number }} + - name: Prune old CI docker images + uses: ./.github/actions/docker-prune-images if: always() - name: Cleanup dangling docker images uses: ./.github/actions/docker-clean-dangling